Cached ARC Producer-Consumer Matrix Workflow #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cached ARC Producer-Consumer Matrix Workflow | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| org_name: | |
| description: 'Organization name to fetch repositories from' | |
| default: 'joshyorko' | |
| max_workers: | |
| description: 'Maximum number of parallel workers' | |
| default: '4' | |
| type: string | |
| # Global env for all jobs | |
| env: | |
| # Point to your cloud rccremote (nginx front), public DNS with valid TLS | |
| RCC_REMOTE_ORIGIN: https://rccremote.example.com:443 | |
| # Put holotree/spaces somewhere writeable and cacheable | |
| ROBOCORP_HOME: ${{ github.workspace }}/.rcc_home | |
| jobs: | |
| producer: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.get-matrix.outputs.matrix }} | |
| shard_count: ${{ steps.get-matrix.outputs.shard_count }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Restore RCC binary cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: .tools/rcc/v18.5.0/rcc | |
| key: rcc-bin-${{ runner.os }}-v18.5.0 | |
| - name: Install RCC (if cache miss) | |
| run: | | |
| if [ ! -x .tools/rcc/v18.5.0/rcc ]; then | |
| mkdir -p .tools/rcc/v18.5.0 | |
| curl -sSL https://downloads.robocorp.com/rcc/releases/v18.5.0/linux64/rcc -o .tools/rcc/v18.5.0/rcc | |
| chmod +x .tools/rcc/v18.5.0/rcc | |
| fi | |
| echo "${GITHUB_WORKSPACE}/.tools/rcc/v18.5.0" >> "$GITHUB_PATH" | |
| rcc version | |
| - name: Cache ROBOCORP_HOME | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.ROBOCORP_HOME }} | |
| key: rcc-home-${{ runner.os }}-rccv18.5.0-${{ hashFiles('**/robot.yaml','**/conda.yaml') }} | |
| restore-keys: | | |
| rcc-home-${{ runner.os }}-rccv18.5.0- | |
| rcc-home-${{ runner.os }}- | |
| - name: Disable RCC telemetry | |
| run: rcc config identity -t | |
| - name: Generate input work item for producer | |
| run: | | |
| mkdir -p devdata/work-items-in/input-for-producer | |
| echo '[{"payload": {"org": "${{ inputs.org_name }}"}}]' > devdata/work-items-in/input-for-producer/work-items.json | |
| - name: Run RCC Producer | |
| run: rcc run -t producer -e devdata/env-for-producer.json | |
| env: | |
| ORG_NAME: ${{ inputs.org_name }} | |
| - name: Warm Holotree | |
| run: | | |
| rcc holotree variables -r robot.yaml -s producer || true | |
| rcc holotree variables -r robot.yaml -s consumer || true | |
| rcc holotree variables -r robot.yaml -s reporter || true | |
| rcc holotree list || true | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Generate shards and matrix | |
| run: python3 scripts/generate_shards_and_matrix.py ${{ inputs.max_workers }} | |
| - name: Get matrix configuration | |
| id: get-matrix | |
| run: | | |
| cat output/matrix-output.json | |
| MATRIX=$(cat output/matrix-output.json) | |
| echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT | |
| SHARD_COUNT=$(echo "${MATRIX}" | jq '.matrix.include | length') | |
| echo "shard_count=${SHARD_COUNT}" >> $GITHUB_OUTPUT | |
| - name: Upload producer output for consumers | |
| uses: actions/[email protected] | |
| with: | |
| name: producer-output | |
| path: | | |
| output/shards/ | |
| output/matrix-output.json | |
| retention-days: 1 | |
| consumer: | |
| runs-on: ubuntu-latest | |
| needs: producer | |
| if: needs.producer.outputs.shard_count > 0 | |
| strategy: | |
| matrix: ${{ fromJSON(needs.producer.outputs.matrix).matrix }} | |
| max-parallel: ${{ fromJSON(inputs.max_workers) }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Restore RCC binary cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: .tools/rcc/v18.5.0/rcc | |
| key: rcc-bin-${{ runner.os }}-v18.5.0 | |
| - name: Install RCC (if cache miss) | |
| run: | | |
| if [ ! -x .tools/rcc/v18.5.0/rcc ]; then | |
| mkdir -p .tools/rcc/v18.5.0 | |
| curl -sSL https://downloads.robocorp.com/rcc/releases/v18.5.0/linux64/rcc -o .tools/rcc/v18.5.0/rcc | |
| chmod +x .tools/rcc/v18.5.0/rcc | |
| fi | |
| echo "${GITHUB_WORKSPACE}/.tools/rcc/v18.5.0" >> "$GITHUB_PATH" | |
| rcc version | |
| - name: Cache ROBOCORP_HOME | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.ROBOCORP_HOME }} | |
| key: rcc-home-${{ runner.os }}-rccv18.5.0-${{ hashFiles('**/robot.yaml','**/conda.yaml') }} | |
| restore-keys: | | |
| rcc-home-${{ runner.os }}-rccv18.5.0- | |
| rcc-home-${{ runner.os }}- | |
| - name: Disable RCC telemetry | |
| run: rcc config identity -t | |
| - name: Download sharded work items | |
| uses: actions/[email protected] | |
| with: | |
| name: producer-output | |
| path: output/ | |
| - name: Setup work items for this shard | |
| run: python3 scripts/shard_loader.py | |
| env: | |
| SHARD_ID: ${{ matrix.shard_id }} | |
| - name: Update consumer env file for shard | |
| run: | | |
| SHARD_PATH="output/shards/work-items-shard-${{ matrix.shard_id }}.json" | |
| cat > devdata/env-for-consumer.json <<EOF | |
| { | |
| "RC_WORKITEM_ADAPTER": "FileAdapter", | |
| "RC_WORKITEM_INPUT_PATH": "${SHARD_PATH}", | |
| "RC_WORKITEM_OUTPUT_PATH": "output/consumer-to-reporter/work-items-${{ matrix.shard_id }}.json" | |
| } | |
| EOF | |
| - name: Run RCC Consumer | |
| run: rcc run -t consumer -e devdata/env-for-consumer.json | |
| env: | |
| SHARD_ID: ${{ matrix.shard_id }} | |
| ORG_NAME: ${{ inputs.org_name }} | |
| - name: Upload shard output for reporter | |
| uses: actions/[email protected] | |
| with: | |
| name: shard-output-${{ matrix.shard_id }} | |
| path: | | |
| output/consumer-to-reporter/work-items-${{ matrix.shard_id }}.json | |
| output/repos-shard-${{ matrix.shard_id }}.zip | |
| output/report-shard-${{ matrix.shard_id }}.json | |
| retention-days: 1 | |
| reporter: | |
| runs-on: ubuntu-latest | |
| needs: [producer, consumer] | |
| if: needs.producer.outputs.shard_count > 0 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Restore RCC binary cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: .tools/rcc/v18.5.0/rcc | |
| key: rcc-bin-${{ runner.os }}-v18.5.0 | |
| - name: Install RCC (if cache miss) | |
| run: | | |
| if [ ! -x .tools/rcc/v18.5.0/rcc ]; then | |
| mkdir -p .tools/rcc/v18.5.0 | |
| curl -sSL https://downloads.robocorp.com/rcc/releases/v18.5.0/linux64/rcc -o .tools/rcc/v18.5.0/rcc | |
| chmod +x .tools/rcc/v18.5.0/rcc | |
| fi | |
| echo "${GITHUB_WORKSPACE}/.tools/rcc/v18.5.0" >> "$GITHUB_PATH" | |
| rcc version | |
| - name: Cache ROBOCORP_HOME | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.ROBOCORP_HOME }} | |
| key: rcc-home-${{ runner.os }}-rccv18.5.0-${{ hashFiles('**/robot.yaml','**/conda.yaml') }} | |
| restore-keys: | | |
| rcc-home-${{ runner.os }}-rccv18.5.0- | |
| rcc-home-${{ runner.os }}- | |
| - name: Disable RCC telemetry | |
| run: rcc config identity -t | |
| - name: Download all shard outputs | |
| uses: actions/[email protected] | |
| with: | |
| pattern: shard-output-* | |
| path: shard-outputs/ | |
| merge-multiple: true | |
| - name: Combine consumer outputs for reporter | |
| run: | | |
| mkdir -p output/reporter-input | |
| jq -s 'add' shard-outputs/consumer-to-reporter/work-items-*.json > output/reporter-input/work-items.json | |
| - name: Create reporter environment config | |
| run: | | |
| mkdir -p output/reporter-final | |
| cat > devdata/env-for-reporter.json <<EOF | |
| { | |
| "RC_WORKITEM_ADAPTER": "FileAdapter", | |
| "RC_WORKITEM_INPUT_PATH": "output/reporter-input/work-items.json", | |
| "RC_WORKITEM_OUTPUT_PATH": "output/reporter-final/work-items.json" | |
| } | |
| EOF | |
| - name: Run RCC Reporter | |
| run: rcc run -t reporter -e devdata/env-for-reporter.json | |
| - name: Upload reporter output | |
| uses: actions/[email protected] | |
| with: | |
| name: reporter-output | |
| path: | | |
| output/final_report_*.json | |
| output/log.html | |
| output/reporter-final/ | |
| retention-days: 7 |