From f3ec5228fff28c20e6cef9c49cccde5b5d237746 Mon Sep 17 00:00:00 2001 From: Clark Feusier Date: Thu, 12 Feb 2026 15:12:40 -0800 Subject: [PATCH] ci: Add E2E smoke test workflow Adds a manually-triggered GitHub Action that runs E2E smoke tests on a cloud VM. Reuses existing infrastructure and test orchestration scripts to avoid drift. Results are displayed in the job summary with per-test links to hosted HTML reports. --- .github/workflows/e2e-smoke.yml | 245 ++++++++++++++++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 .github/workflows/e2e-smoke.yml diff --git a/.github/workflows/e2e-smoke.yml b/.github/workflows/e2e-smoke.yml new file mode 100644 index 0000000..4aaf289 --- /dev/null +++ b/.github/workflows/e2e-smoke.yml @@ -0,0 +1,245 @@ +name: E2E Smoke Tests + +on: + workflow_dispatch: + inputs: + branch: + description: 'Branch to test (e.g. origin/main, origin/feature-x)' + required: true + default: 'origin/main' + +env: + VM_NAME: ${{ secrets.GCP_VM_NAME }} + VM_ZONE: ${{ secrets.GCP_VM_ZONE }} + GCP_PROJECT: ${{ secrets.GCP_PROJECT }} + +jobs: + smoke-test: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Auth to GCP + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Setup gcloud + uses: google-github-actions/setup-gcloud@v2 + + - name: Clone relay-harness + run: | + mkdir -p ~/.ssh + echo '${{ secrets.HARNESS_DEPLOY_KEY }}' > ~/.ssh/harness-key + chmod 600 ~/.ssh/harness-key + cat >> ~/.ssh/config << 'EOF' + Host github.com + IdentityFile ~/.ssh/harness-key + StrictHostKeyChecking accept-new + EOF + + git clone git@github.com:No-Instructions/relay-harness.git ~/relay-harness + + - name: Ensure VM is running + run: | + STATUS=$(gcloud compute instances describe "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --format='get(status)' 2>/dev/null || echo "NOT_FOUND") + + if [ "$STATUS" = "NOT_FOUND" ]; then + echo "VM does not exist. Creating via harness infra scripts..." + cd ~/relay-harness + GCP_PROJECT="$GCP_PROJECT" GCP_ZONE="$VM_ZONE" ./infra/gcp-linux-vm.sh create + elif [ "$STATUS" = "TERMINATED" ] || [ "$STATUS" = "STOPPED" ]; then + echo "Starting stopped VM..." + gcloud compute instances start "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" + elif [ "$STATUS" = "RUNNING" ]; then + echo "VM already running" + else + echo "Unexpected VM status: $STATUS" + exit 1 + fi + + echo "Waiting for SSH readiness..." + for i in $(seq 1 60); do + if gcloud compute ssh "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --command="echo ready" 2>/dev/null; then + echo "VM is ready" + break + fi + if [ "$i" -eq 60 ]; then + echo "Timed out waiting for VM SSH" + exit 1 + fi + sleep 5 + done + + - name: Provision VM credentials + run: | + echo '${{ secrets.GCP_SA_KEY }}' > /tmp/sa-key.json + gcloud compute scp /tmp/sa-key.json "$VM_NAME":~/ci-sa-key.json \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" + rm -f /tmp/sa-key.json + + echo '${{ secrets.HARNESS_DEPLOY_KEY }}' > /tmp/deploy-key + chmod 600 /tmp/deploy-key + gcloud compute scp /tmp/deploy-key "$VM_NAME":~/ci-deploy-key \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" + rm -f /tmp/deploy-key + + - name: Run smoke tests on VM + run: | + BRANCH="${{ github.event.inputs.branch }}" + + gcloud compute ssh "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --command="bash -s -- '$BRANCH'" << 'REMOTE_SCRIPT' + set -eo pipefail + BRANCH="$1" + + # Activate GCP service account for gcloud/GCS uploads + gcloud auth activate-service-account --key-file="$HOME/ci-sa-key.json" 2>/dev/null + + # Set up SSH for private repo access + mkdir -p ~/.ssh + cp ~/ci-deploy-key ~/.ssh/ci-deploy-key + chmod 600 ~/.ssh/ci-deploy-key + if ! grep -q 'ci-deploy-key' ~/.ssh/config 2>/dev/null; then + cat >> ~/.ssh/config << 'SSHCFG' + Host github.com + IdentityFile ~/.ssh/ci-deploy-key + StrictHostKeyChecking accept-new + SSHCFG + fi + + # Clone or update relay-plugin (public) + if [ ! -d ~/relay-plugin ]; then + git clone https://github.com/No-Instructions/Relay.git ~/relay-plugin + fi + cd ~/relay-plugin && git fetch --all && git checkout "$BRANCH" 2>/dev/null || git checkout -b "$(basename "$BRANCH")" "$BRANCH" + + # Clone or update relay-harness (private) + if [ ! -d ~/relay-harness ]; then + git clone git@github.com:No-Instructions/relay-harness.git ~/relay-harness + fi + cd ~/relay-harness && git checkout main && git pull + + # Run tests using the harness test-branch.sh orchestrator + export RELAY_PLUGIN_DIR=~/relay-plugin + cd ~/relay-harness + + TEST_EXIT=0 + ./scripts/test-branch.sh "$BRANCH" --upload || TEST_EXIT=$? + + # Extract results for the GitHub Actions runner + LATEST=$(ls -td /tmp/test-reports/runs/*/* 2>/dev/null | head -1) + if [ -n "$LATEST" ] && [ -f "$LATEST/summary.json" ]; then + cp "$LATEST/summary.json" ~/test-summary.json + + COMMIT=$(basename "$(dirname "$LATEST")") + EXEC_ID=$(basename "$LATEST") + REPORT_URL="https://storage.googleapis.com/relay-e2e-screenshots/runs/${COMMIT}/${EXEC_ID}/index.html" + echo "{\"reportUrl\": \"$REPORT_URL\", \"commit\": \"$COMMIT\", \"execId\": \"$EXEC_ID\"}" > ~/test-metadata.json + else + echo '{"summary":{"total":0,"pass":0,"fail":0},"tests":[],"state":"error"}' > ~/test-summary.json + echo '{"reportUrl":"","commit":"","execId":""}' > ~/test-metadata.json + fi + + exit $TEST_EXIT + REMOTE_SCRIPT + + - name: Clean up VM credentials + if: always() + run: | + gcloud compute ssh "$VM_NAME" \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" \ + --command="rm -f ~/ci-sa-key.json ~/ci-deploy-key ~/.ssh/ci-deploy-key" 2>/dev/null || true + + - name: Fetch results + if: always() + run: | + gcloud compute scp "$VM_NAME":~/test-summary.json ./summary.json \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" 2>/dev/null || true + gcloud compute scp "$VM_NAME":~/test-metadata.json ./metadata.json \ + --zone="$VM_ZONE" --project="$GCP_PROJECT" 2>/dev/null || true + + - name: Generate job summary + if: always() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + + let summary, metadata; + try { + summary = JSON.parse(fs.readFileSync('summary.json', 'utf-8')); + } catch { + core.summary.addRaw('## E2E Smoke Test Results\n\n:x: **Failed to retrieve test results from VM**\n'); + await core.summary.write(); + core.setFailed('No test results available'); + return; + } + try { + metadata = JSON.parse(fs.readFileSync('metadata.json', 'utf-8')); + } catch { + metadata = { reportUrl: '', commit: '', execId: '' }; + } + + const s = summary.summary || {}; + const tests = summary.tests || []; + const reportBase = (metadata.reportUrl || '').replace(/\/index\.html$/, ''); + + // Status emoji map + const statusEmoji = { + pass: ':white_check_mark:', + fail: ':x:', + expected_fail: ':warning:', + unexpected_pass: ':sparkles:', + }; + + // Verdict + const totalDur = Math.round((s.duration || 0) / 1000); + const verdictEmoji = (s.fail || 0) > 0 ? ':x:' : ':white_check_mark:'; + const verdictText = (s.fail || 0) > 0 ? 'FAIL' : 'PASS'; + const parts = []; + if (s.pass) parts.push(`${s.pass} pass`); + if (s.fail) parts.push(`${s.fail} fail`); + if (s.expectedFail) parts.push(`${s.expectedFail} expected-fail`); + if (s.unexpectedPass) parts.push(`${s.unexpectedPass} unexpected-pass`); + + let md = `## E2E Smoke Test Results\n\n`; + md += `${verdictEmoji} **${verdictText}** | ${parts.join(', ')} | ${totalDur}s`; + if (metadata.reportUrl) { + md += ` | [Full Report](${metadata.reportUrl})`; + } + md += `\n\n`; + + // Test table + md += `| Test | Status | Assertions | Duration |\n`; + md += `|------|--------|------------|----------|\n`; + for (const t of tests) { + const emoji = statusEmoji[t.status] || ':grey_question:'; + const dur = `${(t.duration / 1000).toFixed(1)}s`; + const assertions = `${t.passedAssertions ?? '?'}/${t.assertions ?? '?'}`; + const testUrl = reportBase ? `${reportBase}/${t.dir}/index.html` : ''; + const name = testUrl ? `[${t.name}](${testUrl})` : t.name; + + let row = `| ${name} | ${emoji} ${t.status} | ${assertions} | ${dur} |`; + md += row + '\n'; + + // Add error detail for failures + if (t.status === 'fail' && t.firstError) { + const errText = t.firstError.split('\n')[0].substring(0, 120); + md += `| | | \`${errText}\` | |\n`; + } + } + + core.summary.addRaw(md); + await core.summary.write(); + + // Set job status + if ((s.fail || 0) > 0) { + core.setFailed(`${s.fail} test(s) failed`); + }