fix: add basic error handling to manifest-server calls #3925
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Connectors | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| # TODO: Consider moving to run these only after the "PyTest (Fast)" workflow is successful. | |
| # workflow_run: | |
| # workflows: [PyTest (Fast)] | |
| # types: | |
| # - completed | |
| concurrency: | |
| # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow. | |
| # | |
| # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests | |
| # ensuring that only one run per pull request is active at a time. | |
| # | |
| # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches. | |
| # This allows us to run multiple workflow dispatches in parallel. | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| cdk_changes: | |
| name: Get Changes | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| statuses: write | |
| pull-requests: read | |
| steps: | |
| - name: Checkout Airbyte | |
| if: github.event_name != 'pull_request' | |
| uses: actions/checkout@v4 | |
| - id: changes | |
| uses: dorny/[email protected] | |
| with: | |
| filters: | | |
| src: | |
| - 'airbyte_cdk/**' | |
| - 'bin/**' | |
| - 'poetry.lock' | |
| - 'pyproject.toml' | |
| file-based: | |
| - 'airbyte_cdk/sources/file_based/**' | |
| vector-db-based: | |
| - 'airbyte_cdk/destinations/vector_db_based/**' | |
| sql: | |
| - 'airbyte_cdk/sql/**' | |
| outputs: | |
| # Source code modified: | |
| src: ${{ steps.changes.outputs.src }} | |
| # Extras modified: | |
| file-based: ${{ steps.changes.outputs.file-based }} | |
| vector-db-based: ${{ steps.changes.outputs.vector-db-based }} | |
| sql: ${{ steps.changes.outputs.sql }} | |
| connectors_ci: | |
| needs: cdk_changes | |
| # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR | |
| # Forked PRs are handled by the community_ci.yml workflow | |
| # If the condition is not met the job will be skipped (it will not fail) | |
| # runs-on: connector-test-large | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 360 # 6 hours | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - connector: source-hardcoded-records | |
| cdk_extra: n/a | |
| - connector: source-shopify | |
| cdk_extra: n/a | |
| - connector: source-google-drive | |
| cdk_extra: file-based | |
| - connector: destination-motherduck | |
| # For now, we mark as 'n/a' to always test this connector | |
| cdk_extra: n/a # change to 'sql' to test less often | |
| # source-amplitude failing for unrelated issue "date too far back" | |
| # e.g. https://github.com/airbytehq/airbyte-python-cdk/actions/runs/16053716569/job/45302638848?pr=639 | |
| # - connector: source-amplitude | |
| # cdk_extra: n/a | |
| - connector: source-intercom | |
| cdk_extra: n/a | |
| - connector: source-pokeapi | |
| cdk_extra: n/a | |
| # CDK Tests cannot build the Connector object (constructor args not optional). | |
| # - connector: source-zendesk-support | |
| # cdk_extra: n/a | |
| name: "${{ needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' && 'Skipped Check' || 'Check' }}: ${{matrix.connector}}" | |
| if: needs.cdk_changes.outputs['src'] == 'true' | |
| permissions: | |
| checks: write | |
| contents: write # Required for creating commit statuses | |
| pull-requests: read | |
| steps: | |
| - name: Abort if extra not changed (${{matrix.cdk_extra}}) | |
| id: no_changes | |
| if: ${{ needs.cdk_changes.outputs['src'] == 'false' || matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }} | |
| run: | | |
| echo "Aborting job." | |
| echo "Source code changed: ${{ needs.cdk_changes.outputs['src'] }}" | |
| if [ "${{ matrix.cdk_extra }}" != "n/a" ]; then | |
| echo "Extra not changed: ${{ matrix.cdk_extra }} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}" | |
| fi | |
| echo "> Skipped '${{matrix.connector}}' (no relevant changes)" >> $GITHUB_STEP_SUMMARY | |
| echo "status=cancelled" >> $GITHUB_OUTPUT | |
| exit 0 | |
| continue-on-error: true | |
| # Get the monorepo so we can test the connectors | |
| - name: Checkout CDK | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| uses: actions/checkout@v4 | |
| with: | |
| path: airbyte-python-cdk | |
| - name: Checkout Airbyte Monorepo | |
| uses: actions/checkout@v4 | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| with: | |
| repository: airbytehq/airbyte | |
| ref: master | |
| path: airbyte | |
| - name: Set up Python | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Set up `uv` | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| uses: astral-sh/[email protected] | |
| - name: Set up `poe` | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| run: | | |
| uv tool install poethepoet | |
| - name: Set up Poetry | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: "2.0.1" | |
| - name: Get Connector Language | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| working-directory: airbyte/airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: | | |
| # Get the language of the connector from the metadata file | |
| CONNECTOR_LANGUAGE=$(poe -qq get-language) | |
| echo "CONNECTOR_LANGUAGE=$CONNECTOR_LANGUAGE" | tee -a $GITHUB_ENV | |
| - name: Install CDK with Poetry | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| run: | | |
| cd airbyte-python-cdk | |
| poetry install --all-extras | |
| - name: Fetch Connector Secrets | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| timeout-minutes: 90 | |
| working-directory: airbyte-python-cdk | |
| env: | |
| GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" | |
| run: | | |
| poetry run airbyte-cdk secrets fetch ${{ matrix.connector }} | |
| - name: Bump to Dev Branch CDK [Python Connectors] | |
| if: env.CONNECTOR_LANGUAGE == 'python' | |
| working-directory: airbyte/airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: | | |
| echo "Using CDK ref ${{ github.event.pull_request.head.sha || github.sha }}" | |
| poe use-cdk-branch ${{ github.event.pull_request.head.sha || github.sha }} | |
| poetry install --all-extras | |
| - name: Run Unit Tests [Python Connectors] | |
| if: env.CONNECTOR_LANGUAGE == 'python' | |
| working-directory: airbyte/airbyte-integrations/connectors/${{ matrix.connector }} | |
| run: | | |
| poe test-unit-tests | |
| - name: Run FAST Standard Tests [Python Connectors] | |
| if: env.CONNECTOR_LANGUAGE == 'python' | |
| working-directory: airbyte/airbyte-integrations/connectors/${{ matrix.connector }} | |
| env: | |
| GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" | |
| run: | | |
| poetry run airbyte-cdk connector test | |
| - name: Run FAST Standard Tests [Manifest-Only Connectors] | |
| if: env.CONNECTOR_LANGUAGE == 'manifest-only' | |
| working-directory: airbyte-python-cdk | |
| env: | |
| GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" | |
| run: | | |
| poetry run airbyte-cdk connector test ${{ matrix.connector }} | |
| - name: Container Build and Test [All Connectors] | |
| if: steps.no_changes.outputs.status != 'cancelled' | |
| working-directory: airbyte-python-cdk | |
| env: | |
| GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" | |
| run: | | |
| poetry run airbyte-cdk image test ${{ matrix.connector }} |