diff --git a/.github/workflows/_accuracy_test.yml b/.github/workflows/_accuracy_test.yml index 72f1e609c37..9ccb2b57fe2 100644 --- a/.github/workflows/_accuracy_test.yml +++ b/.github/workflows/_accuracy_test.yml @@ -44,7 +44,7 @@ jobs: FULL_REPO="${{ github.repository }}" REPO_NAME="${FULL_REPO##*/}" BASE_BRANCH="${{ github.base_ref }}" - + docker pull ${docker_image} # Clean the repository directory before starting docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ -e "REPO_NAME=${REPO_NAME}" \ diff --git a/.github/workflows/_base_test.yml b/.github/workflows/_base_test.yml index be369632071..de1233a417d 100644 --- a/.github/workflows/_base_test.yml +++ b/.github/workflows/_base_test.yml @@ -44,7 +44,7 @@ jobs: FULL_REPO="${{ github.repository }}" REPO_NAME="${FULL_REPO##*/}" BASE_BRANCH="${{ github.base_ref }}" - + docker pull ${docker_image} # Clean the repository directory before starting docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ -e "REPO_NAME=${REPO_NAME}" \ diff --git a/.github/workflows/_ci_image_build.yml b/.github/workflows/_ci_image_build.yml new file mode 100644 index 00000000000..92ff8b6077c --- /dev/null +++ b/.github/workflows/_ci_image_build.yml @@ -0,0 +1,73 @@ +name: Docker Build +description: "FastDeploy CI Image Build" + +on: + workflow_call: + inputs: + CI_DOCKER_IMAGE_NAME: + description: "Build Images" + required: true + type: string + default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310" + FASTDEPLOY_ARCHIVE_URL: + description: "URL of the compressed FastDeploy code archive." + required: true + type: string + DOCKER_IMAGE_NAME: + description: "Build Images" + required: false + type: string + default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate" + outputs: + docker_name_precheck: + description: "Output path of the generated wheel" + value: ${{ jobs.docker_build.outputs.docker_name_precheck }} + +jobs: + docker_build: + runs-on: [self-hosted, Docker-Build] + outputs: + docker_name_precheck: ${{ steps.docker_build.outputs.docker_name_precheck }} + steps: + - name: Code Prepare + id: docker_build + shell: bash + env: + docker_image_name: ${{ inputs.CI_DOCKER_IMAGE_NAME }} + docker_image: ${{ inputs.DOCKER_IMAGE_NAME }} + fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }} + run: | + set -x + REPO="https://github.com/${{ github.repository }}.git" + FULL_REPO="${{ github.repository }}" + REPO_NAME="${FULL_REPO##*/}" + BASE_BRANCH="${{ github.base_ref }}" + + # Clean the repository directory before starting + docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ + -e "REPO_NAME=${REPO_NAME}" \ + ${docker_image} /bin/bash -c ' + if [ -d ${REPO_NAME} ]; then + echo "Directory ${REPO_NAME} exists, removing it..." + rm -rf ${REPO_NAME}* + fi + ' + + wget -q ${fd_archive_url} + tar -xf FastDeploy.tar.gz + rm -rf FastDeploy.tar.gz + cd FastDeploy + git config --global user.name "FastDeployCI" + git config --global user.email "fastdeploy_ci@example.com" + git log -n 3 --oneline + + # Docker Build + cd tools/dockerfile/ + set -e + cp ../../requirements.txt ./ + cp ../../scripts/unittest_requirement.txt ./ + docker build -t ${docker_image_name} -f Dockerfile.ci . \ + --network host \ + --no-cache + docker push ${docker_image_name} + echo "docker_name_precheck=${docker_image_name}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/_logprob_test_linux.yml b/.github/workflows/_logprob_test_linux.yml index 435ecf81eb6..138f2b2bc2e 100644 --- a/.github/workflows/_logprob_test_linux.yml +++ b/.github/workflows/_logprob_test_linux.yml @@ -39,6 +39,7 @@ jobs: docker_image: ${{ inputs.DOCKER_IMAGE }} paddletest_archive_url: ${{ inputs.PADDLETEST_ARCHIVE_URL }} run: | + docker pull ${docker_image} # Clean the repository directory before starting docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ -e "REPO_NAME=${REPO_NAME}" \ @@ -114,7 +115,6 @@ jobs: echo "Removing stale container: ${runner_name}" docker rm -f ${runner_name} || true fi - docker run --rm --ipc=host --pid=host --net=host \ --name ${runner_name} \ -v $(pwd):/workspace \ diff --git a/.github/workflows/_pre_ce_test.yml b/.github/workflows/_pre_ce_test.yml index 5ed0ad3534b..89e67567207 100644 --- a/.github/workflows/_pre_ce_test.yml +++ b/.github/workflows/_pre_ce_test.yml @@ -46,7 +46,7 @@ jobs: FULL_REPO="${{ github.repository }}" REPO_NAME="${FULL_REPO##*/}" BASE_BRANCH="${{ github.base_ref }}" - + docker pull ${docker_image} # Clean the repository directory before starting docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ -e "REPO_NAME=${REPO_NAME}" \ diff --git a/.github/workflows/_stable_test.yml b/.github/workflows/_stable_test.yml index 5988f5a735d..b45695bb15b 100644 --- a/.github/workflows/_stable_test.yml +++ b/.github/workflows/_stable_test.yml @@ -44,7 +44,7 @@ jobs: FULL_REPO="${{ github.repository }}" REPO_NAME="${FULL_REPO##*/}" BASE_BRANCH="${{ github.base_ref }}" - + docker pull ${docker_image} # Clean the repository directory before starting docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ -e "REPO_NAME=${REPO_NAME}" \ diff --git a/.github/workflows/_unit_test_coverage.yml b/.github/workflows/_unit_test_coverage.yml index 6df21ffbf05..132d0ab1565 100644 --- a/.github/workflows/_unit_test_coverage.yml +++ b/.github/workflows/_unit_test_coverage.yml @@ -60,7 +60,7 @@ jobs: FULL_REPO="${{ github.repository }}" REPO_NAME="${FULL_REPO##*/}" BASE_BRANCH="${{ github.base_ref }}" - + docker pull ${docker_image} # Clean the repository directory before starting docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ -e "REPO_NAME=${REPO_NAME}" \ diff --git a/.github/workflows/ci_image_update.yml b/.github/workflows/ci_image_update.yml new file mode 100644 index 00000000000..e01e5e8e16b --- /dev/null +++ b/.github/workflows/ci_image_update.yml @@ -0,0 +1,174 @@ +name: CI Images Build + +on: + workflow_dispatch: + schedule: + - cron: '0 18 * * *' # 2:00 AM China Standard Time (UTC+8) + +permissions: read-all + +concurrency: + group: ${{ github.ref }}-${{ github.sha }} + cancel-in-progress: true + + +jobs: + clone: + environment: CodeSync + name: FD-Clone-Linux + runs-on: ubuntu-latest + outputs: + repo_archive_url: ${{ steps.set_output.outputs.repo_archive_url }} + steps: + - name: Clone FastDeploy + uses: actions/checkout@v4 + with: + ref: ${{ github.ref_name }} + submodules: 'recursive' + fetch-depth: 1000 + + - name: Python Setup + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Code Info Show and Upload + id: set_output + env: + AK: ${{ secrets.BOS_AK }} + SK: ${{ secrets.BOS_SK }} + run: | + git config --unset http.https://github.com/.extraheader + git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'" + git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'" + echo "Current HEAD Log:" + git log --oneline -n 5 + ls + cd .. + tar -zcf FastDeploy.tar.gz FastDeploy + if [[ "${{ github.ref_type }}" == "tag" ]]; then + commit_id=${{ github.sha }} + tag_name=${{ github.ref_name }} + target_path=paddle-qa/TAG/FastDeploy/${tag_name}/${commit_id} + else + commit_id=${{ github.sha }} + branch_name=${{ github.ref_name }} + target_path=paddle-qa/BRANCH/FastDeploy/${branch_name}/${commit_id} + fi + wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py + push_file=$(realpath bos_tools.py) + python -m pip install bce-python-sdk==0.9.29 + ls + python ${push_file} FastDeploy.tar.gz ${target_path} + target_path_stripped="${target_path#paddle-qa/}" + REPO_ARCHIVE_URL=https://paddle-qa.bj.bcebos.com/${target_path_stripped}/FastDeploy.tar.gz + echo "repo_archive_url=${REPO_ARCHIVE_URL}" >> $GITHUB_OUTPUT + + resultshow: + name: Show Code Archive Output + needs: clone + runs-on: ubuntu-latest + steps: + - name: Print wheel path + run: | + echo "The code archive is located at: ${{ needs.clone.outputs.repo_archive_url }}" + + ci_image_build: + name: CI Images Build + needs: clone + uses: ./.github/workflows/_ci_image_build.yml + with: + CI_DOCKER_IMAGE_NAME: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate-precheck + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + + + build_sm8090: + name: BUILD_SM8090 + needs: [clone, ci_image_build] + uses: ./.github/workflows/_build_linux.yml + with: + DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + COMPILE_ARCH: "90" + WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }} + FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }} + PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }} + PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }} + + + unittest_coverage: + name: Run FastDeploy Unit Tests and Coverage + needs: [clone,build_sm8090,ci_image_build] + uses: ./.github/workflows/_unit_test_coverage.yml + with: + DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" + secrets: + github-token: ${{ secrets.GITHUB_TOKEN }} + + logprob_test: + name: Run FastDeploy LogProb Tests + needs: [build_sm8090,ci_image_build] + uses: ./.github/workflows/_logprob_test_linux.yml + with: + DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} + PADDLETEST_ARCHIVE_URL: "https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz" + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" + + pre_ce_test: + name: Extracted partial CE model tasks to run in CI. + needs: [clone,build_sm8090,ci_image_build] + uses: ./.github/workflows/_pre_ce_test.yml + with: + DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" + + base_test: + name: Run Base Tests + needs: [clone,build_sm8090,ci_image_build] + uses: ./.github/workflows/_base_test.yml + with: + DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" + + accuracy_test: + name: Run Accuracy Tests + needs: [clone,build_sm8090,ci_image_build] + uses: ./.github/workflows/_accuracy_test.yml + with: + DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" + + stable_test: + name: Run Stable Tests + needs: [clone,build_sm8090,ci_image_build] + uses: ./.github/workflows/_stable_test.yml + with: + DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" + + + publish_pre_check: + name: Publish Docker Images Pre Check + needs: [ci_image_build, unittest_coverage,logprob_test,pre_ce_test,base_test,accuracy_test,stable_test] + runs-on: [self-hosted, Docker-Build] + steps: + - name: Images Uploading + env: + images_name: ${{ needs.ci_image_build.outputs.docker_name_precheck }} + ci_image_name: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate" + run: | + echo "images_name=${images_name}" + docker images ${ci_image_name} + docker tag ${images_name} ${ci_image_name} + docker push ${ci_image_name} diff --git a/.github/workflows/pr_build_and_test.yml b/.github/workflows/pr_build_and_test.yml index 9e12c89da60..23eb2fefa89 100644 --- a/.github/workflows/pr_build_and_test.yml +++ b/.github/workflows/pr_build_and_test.yml @@ -21,7 +21,7 @@ jobs: with: DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} - COMPILE_ARCH: "89,90" + COMPILE_ARCH: "90" WITH_NIGHTLY_BUILD: "OFF" FD_VERSION: "0.0.0" diff --git a/.github/workflows/publish_job.yml b/.github/workflows/publish_job.yml index 5e98d092762..1ed9b1863b3 100644 --- a/.github/workflows/publish_job.yml +++ b/.github/workflows/publish_job.yml @@ -319,3 +319,13 @@ jobs: FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" + + stable_test: + name: Run Stable Tests + needs: [clone,build] + uses: ./.github/workflows/_stable_test.yml + with: + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" diff --git a/scripts/unittest_requirement.txt b/scripts/unittest_requirement.txt index a7c8066f29c..52b0c03f7d5 100644 --- a/scripts/unittest_requirement.txt +++ b/scripts/unittest_requirement.txt @@ -1,9 +1,6 @@ respx pytest pytest-asyncio -pytest-tornasync -pytest-trio -pytest-twisted anyio coverage diff-cover diff --git a/tools/dockerfile/Dockerfile.ci b/tools/dockerfile/Dockerfile.ci index 197efb3ffc1..32fb95e0180 100644 --- a/tools/dockerfile/Dockerfile.ci +++ b/tools/dockerfile/Dockerfile.ci @@ -1,3 +1,6 @@ FROM ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310-cibase +RUN apt update && apt install -y jq COPY ../../requirements.txt ./requirements.txt RUN python -m pip install -r requirements.txt -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple && python -m pip install pytest +COPY ./unittest_requirement.txt ./unittest_requirement.txt +RUN python -m pip install -r unittest_requirement.txt -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple diff --git a/tools/dockerfile/docker_build.sh b/tools/dockerfile/docker_build.sh index d8e5f0ab55e..6794d80a323 100644 --- a/tools/dockerfile/docker_build.sh +++ b/tools/dockerfile/docker_build.sh @@ -1,9 +1,5 @@ PRODUCT_NAME='ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:fastdeploy-ciuse-cuda126' -#proxy='agent.baidu.com:8188' cp ../../requirements.txt ./ - +cp ../../scripts/unittest_requirement.txt ./ docker build -t ${PRODUCT_NAME} -f Dockerfile.ci . \ --network host - # --build-arg HTTP_PROXY=${proxy} \ - # --build-arg HTTPS_PROXY=${proxy} \ - # --build-arg ftp_proxy=${proxy} diff --git a/tools/dockerfile/requirements.txt b/tools/dockerfile/requirements.txt deleted file mode 100644 index 2c5215edead..00000000000 --- a/tools/dockerfile/requirements.txt +++ /dev/null @@ -1,27 +0,0 @@ -setuptools==75.1.0 -pre-commit -yapf -flake8 -ruamel.yaml -zmq -aiozmq -openai -tqdm -pynvml -uvicorn -fastapi -paddlenlp @ https://paddle-qa.bj.bcebos.com/paddlenlp/wheel/c1322be06cdf366ad2d996e963499f5298d2a2bc/paddlenlp-3.0.0b4.post20250530-py3-none-any.whl -redis -etcd3 -httpx -tool_helpers -cupy-cuda12x -pybind11 -tabulate -gradio -xlwt -visualdl -setuptools-scm>=8 -prometheus-client -decord -moviepy