CI Images Build #29
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Images Build | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 18 * * *' # 2:00 AM China Standard Time (UTC+8) | |
| permissions: read-all | |
| concurrency: | |
| group: CI-Images-Build-${{ github.ref }}-${{ github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| clone: | |
| environment: CodeSync | |
| name: FD-Clone-Linux | |
| runs-on: ubuntu-latest | |
| outputs: | |
| repo_archive_url: ${{ steps.set_output.outputs.repo_archive_url }} | |
| steps: | |
| - name: Clone FastDeploy | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref_name }} | |
| submodules: 'recursive' | |
| fetch-depth: 1000 | |
| - name: Python Setup | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Code Info Show and Upload | |
| id: set_output | |
| env: | |
| AK: ${{ secrets.BOS_AK }} | |
| SK: ${{ secrets.BOS_SK }} | |
| run: | | |
| git config --unset http.https://github.com/.extraheader | |
| git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'" | |
| git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'" | |
| echo "Current HEAD Log:" | |
| git log --oneline -n 5 | |
| ls | |
| cd .. | |
| tar -zcf FastDeploy.tar.gz FastDeploy | |
| if [[ "${{ github.ref_type }}" == "tag" ]]; then | |
| commit_id=${{ github.sha }} | |
| tag_name=${{ github.ref_name }} | |
| target_path=paddle-qa/TAG/FastDeploy/${tag_name}/${commit_id} | |
| else | |
| commit_id=${{ github.sha }} | |
| branch_name=${{ github.ref_name }} | |
| target_path=paddle-qa/BRANCH/FastDeploy/${branch_name}/${commit_id} | |
| fi | |
| wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py | |
| push_file=$(realpath bos_tools.py) | |
| python -m pip install bce-python-sdk==0.9.29 | |
| ls | |
| python ${push_file} FastDeploy.tar.gz ${target_path} | |
| target_path_stripped="${target_path#paddle-qa/}" | |
| REPO_ARCHIVE_URL=https://paddle-qa.bj.bcebos.com/${target_path_stripped}/FastDeploy.tar.gz | |
| echo "repo_archive_url=${REPO_ARCHIVE_URL}" >> $GITHUB_OUTPUT | |
| resultshow: | |
| name: Show Code Archive Output | |
| needs: clone | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Print wheel path | |
| run: | | |
| echo "The code archive is located at: ${{ needs.clone.outputs.repo_archive_url }}" | |
| ci_image_build: | |
| name: CI Images Build | |
| needs: clone | |
| uses: ./.github/workflows/_ci_image_build.yml | |
| with: | |
| CI_DOCKER_IMAGE_NAME: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate-precheck | |
| FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} | |
| build_sm8090: | |
| name: BUILD_SM8090 | |
| needs: [clone, ci_image_build] | |
| uses: ./.github/workflows/_build_linux.yml | |
| with: | |
| DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} | |
| FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} | |
| COMPILE_ARCH: "90" | |
| WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }} | |
| FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }} | |
| PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }} | |
| PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }} | |
| unittest_coverage: | |
| name: Run FastDeploy Unit Tests and Coverage | |
| needs: [clone,build_sm8090,ci_image_build] | |
| uses: ./.github/workflows/_unit_test_coverage.yml | |
| with: | |
| DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} | |
| FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} | |
| FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} | |
| MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" | |
| secrets: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| logprob_test: | |
| name: Run FastDeploy LogProb Tests | |
| needs: [build_sm8090,ci_image_build] | |
| uses: ./.github/workflows/_logprob_test_linux.yml | |
| with: | |
| DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} | |
| PADDLETEST_ARCHIVE_URL: "https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz" | |
| FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} | |
| MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" | |
| pre_ce_test: | |
| name: Extracted partial CE model tasks to run in CI. | |
| needs: [clone,build_sm8090,ci_image_build] | |
| uses: ./.github/workflows/_pre_ce_test.yml | |
| with: | |
| DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} | |
| FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} | |
| FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} | |
| MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" | |
| base_test: | |
| name: Run Base Tests | |
| needs: [clone,build_sm8090,ci_image_build] | |
| uses: ./.github/workflows/_base_test.yml | |
| with: | |
| DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} | |
| FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} | |
| FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} | |
| MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" | |
| accuracy_test: | |
| name: Run Accuracy Tests | |
| needs: [clone,build_sm8090,ci_image_build] | |
| uses: ./.github/workflows/_accuracy_test.yml | |
| with: | |
| DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} | |
| FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} | |
| FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} | |
| MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" | |
| stable_test: | |
| name: Run Stable Tests | |
| needs: [clone,build_sm8090,ci_image_build] | |
| uses: ./.github/workflows/_stable_test.yml | |
| with: | |
| DOCKER_IMAGE: ${{ needs.ci_image_build.outputs.docker_name_precheck }} | |
| FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} | |
| FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} | |
| MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" | |
| publish_pre_check: | |
| name: Publish Docker Images Pre Check | |
| needs: [ci_image_build, unittest_coverage,logprob_test,pre_ce_test,base_test,accuracy_test,stable_test] | |
| runs-on: [self-hosted, Docker-Build] | |
| steps: | |
| - name: Images Uploading | |
| env: | |
| images_name: ${{ needs.ci_image_build.outputs.docker_name_precheck }} | |
| ci_image_name: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate" | |
| run: | | |
| echo "images_name=${images_name}" | |
| docker images ${ci_image_name} | |
| docker tag ${images_name} ${ci_image_name} | |
| docker push ${ci_image_name} |