From d0c44e8faf2df39840582ecf52d459072703d948 Mon Sep 17 00:00:00 2001 From: Johnpaul Date: Fri, 17 Oct 2025 02:51:59 +0100 Subject: [PATCH 1/6] feat: Refactor Playwright expectations update workflow for improved caching and sharded snapshot handling --- .../pr-update-playwright-expectations.yaml | 147 ++++++++++++++---- 1 file changed, 120 insertions(+), 27 deletions(-) diff --git a/.github/workflows/pr-update-playwright-expectations.yaml b/.github/workflows/pr-update-playwright-expectations.yaml index f688c3250a..5b20b1f5f1 100644 --- a/.github/workflows/pr-update-playwright-expectations.yaml +++ b/.github/workflows/pr-update-playwright-expectations.yaml @@ -12,11 +12,11 @@ concurrency: cancel-in-progress: true jobs: - test: + setup: runs-on: ubuntu-latest if: > ( github.event_name == 'pull_request' && github.event.label.name == 'New Browser Test Expectations' ) || - ( github.event.issue.pull_request && + ( github.event.issue.pull_request && github.event_name == 'issue_comment' && ( github.event.comment.author_association == 'OWNER' || @@ -24,12 +24,25 @@ jobs: github.event.comment.author_association == 'COLLABORATOR' ) && startsWith(github.event.comment.body, '/update-playwright') ) + outputs: + cache-key: ${{ steps.cache-key.outputs.key }} + pr-number: ${{ steps.pr-info.outputs.pr-number }} + branch: ${{ steps.pr-info.outputs.branch }} + comment-id: ${{ steps.find-update-comment.outputs.comment-id }} steps: + - name: Get PR info + id: pr-info + run: | + echo "pr-number=${{ github.event.number || github.event.issue.number }}" >> $GITHUB_OUTPUT + echo "branch=$(gh pr view ${{ github.event.number || github.event.issue.number }} --repo ${{ github.repository }} --json headRefName --jq '.headRefName')" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Find Update Comment uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad id: "find-update-comment" with: - issue-number: ${{ github.event.number || github.event.issue.number }} + issue-number: ${{ steps.pr-info.outputs.pr-number }} comment-author: "github-actions[bot]" body-includes: "Updating Playwright Expectations" @@ -37,49 +50,129 @@ jobs: uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 with: comment-id: ${{ steps.find-update-comment.outputs.comment-id }} - issue-number: ${{ github.event.number || github.event.issue.number }} + issue-number: ${{ steps.pr-info.outputs.pr-number }} body: | Updating Playwright Expectations edit-mode: replace reactions: eyes - - name: Get Branch SHA - id: "get-branch" - run: echo ::set-output name=branch::$(gh pr view $PR_NO --repo $REPO --json headRefName --jq '.headRefName') - env: - REPO: ${{ github.repository }} - PR_NO: ${{ github.event.number || github.event.issue.number }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Initial Checkout + - name: Checkout repository uses: actions/checkout@v5 with: - ref: ${{ steps.get-branch.outputs.branch }} - - name: Setup Frontend + ref: ${{ steps.pr-info.outputs.branch }} + + # Setup Test Environment, build frontend but do not start server yet + - name: Setup ComfyUI server + uses: ./.github/actions/setup-comfyui-server + - name: Setup frontend uses: ./.github/actions/setup-frontend with: include_build_step: true - - name: Setup ComfyUI Server + - name: Setup Playwright + uses: ./.github/actions/setup-playwright + + # Save the entire workspace as cache for update jobs to restore + - name: Generate cache key + id: cache-key + run: echo "key=$(date +%s)" >> $GITHUB_OUTPUT + - name: Save cache + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 + with: + path: . + key: comfyui-setup-${{ steps.cache-key.outputs.key }} + + # Sharded snapshot updates + update-snapshots-sharded: + needs: setup + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2, 3, 4] + shardTotal: [4] + steps: + # Restore test environment from setup job + - name: Wait for cache propagation + run: sleep 10 + + - name: Restore cached setup + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 + with: + fail-on-cache-miss: true + path: . + key: comfyui-setup-${{ needs.setup.outputs.cache-key }} + + - name: Setup ComfyUI server uses: ./.github/actions/setup-comfyui-server with: launch_server: true + + - name: Setup nodejs, pnpm, reuse built frontend + uses: ./.github/actions/setup-frontend + - name: Setup Playwright uses: ./.github/actions/setup-playwright - - name: Run Playwright tests and update snapshots + + # Run sharded tests with snapshot updates + - name: Update snapshots (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}) id: playwright-tests - run: pnpm exec playwright test --update-snapshots + run: pnpm exec playwright test --update-snapshots --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} continue-on-error: true - - uses: actions/upload-artifact@v4 + + # Upload updated snapshots from this shard + - name: Upload updated snapshots + uses: actions/upload-artifact@v4 + if: always() + with: + name: snapshots-shard-${{ matrix.shardIndex }} + path: browser_tests/**/*-snapshots/ + retention-days: 1 + + - name: Upload test report + uses: actions/upload-artifact@v4 if: always() with: - name: playwright-report + name: playwright-report-shard-${{ matrix.shardIndex }} path: ./playwright-report/ retention-days: 30 + + # Merge snapshots and commit + merge-and-commit: + needs: [setup, update-snapshots-sharded] + runs-on: ubuntu-latest + if: always() + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ needs.setup.outputs.branch }} + + # Download all snapshot artifacts from shards + - name: Download all snapshots + uses: actions/download-artifact@v4 + with: + pattern: snapshots-shard-* + path: ./downloaded-snapshots + merge-multiple: false + + # Merge snapshots from all shards into browser_tests directory + - name: Merge snapshots + run: | + echo "Merging snapshots from all shards..." + for shard_dir in ./downloaded-snapshots/snapshots-shard-*; do + if [ -d "$shard_dir" ]; then + echo "Copying from $shard_dir..." + cp -r "$shard_dir"/* browser_tests/ 2>/dev/null || true + fi + done + echo "Snapshot merge complete" + - name: Debugging info run: | - echo "PR: ${{ github.event.issue.number }}" - echo "Branch: ${{ steps.get-branch.outputs.branch }}" + echo "PR: ${{ needs.setup.outputs.pr-number }}" + echo "Branch: ${{ needs.setup.outputs.branch }}" git status + - name: Commit updated expectations run: | git config --global user.name 'github-actions' @@ -89,20 +182,20 @@ jobs: echo "No changes to commit" else git commit -m "[automated] Update test expectations" - git push origin ${{ steps.get-branch.outputs.branch }} + git push origin ${{ needs.setup.outputs.branch }} fi - name: Add Done Reaction uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 if: github.event_name == 'issue_comment' with: - comment-id: ${{ steps.find-update-comment.outputs.comment-id }} - issue-number: ${{ github.event.number || github.event.issue.number }} + comment-id: ${{ needs.setup.outputs.comment-id }} + issue-number: ${{ needs.setup.outputs.pr-number }} reactions: +1 reactions-edit-mode: replace - name: Remove New Browser Test Expectations label if: always() && github.event_name == 'pull_request' - run: gh pr edit ${{ github.event.pull_request.number }} --remove-label "New Browser Test Expectations" + run: gh pr edit ${{ needs.setup.outputs.pr-number }} --remove-label "New Browser Test Expectations" env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From a05ac48735b59f87a96ed9a102f264ac4ec8c5cb Mon Sep 17 00:00:00 2001 From: Johnpaul Date: Tue, 21 Oct 2025 06:29:13 +0100 Subject: [PATCH 2/6] fix workflow issues and create modify ui to test snapshot --- .../actions/setup-comfyui-server/action.yml | 6 ++ .../pr-update-playwright-expectations.yaml | 72 ++++++++++++++----- src/components/graph/SelectionToolbox.vue | 2 +- 3 files changed, 60 insertions(+), 20 deletions(-) diff --git a/.github/actions/setup-comfyui-server/action.yml b/.github/actions/setup-comfyui-server/action.yml index d1aa1bd578..7aaf9220bd 100644 --- a/.github/actions/setup-comfyui-server/action.yml +++ b/.github/actions/setup-comfyui-server/action.yml @@ -9,6 +9,10 @@ inputs: description: 'Whether to launch the server after setup' required: false default: 'false' + skip_checkout: + description: 'Skip checking out ComfyUI repo (use when restoring from cache)' + required: false + default: 'false' runs: using: 'composite' steps: @@ -16,6 +20,7 @@ runs: # Checkout ComfyUI repo, install the dev_tools node and start server - name: Checkout ComfyUI + if: ${{ inputs.skip_checkout != 'true' }} uses: actions/checkout@v5 with: repository: 'comfyanonymous/ComfyUI' @@ -38,6 +43,7 @@ runs: python-version: '3.10' - name: Install Python requirements + if: ${{ inputs.skip_checkout != 'true' }} shell: bash working-directory: ComfyUI run: | diff --git a/.github/workflows/pr-update-playwright-expectations.yaml b/.github/workflows/pr-update-playwright-expectations.yaml index 5b20b1f5f1..2e6d3df96a 100644 --- a/.github/workflows/pr-update-playwright-expectations.yaml +++ b/.github/workflows/pr-update-playwright-expectations.yaml @@ -60,25 +60,22 @@ jobs: uses: actions/checkout@v5 with: ref: ${{ steps.pr-info.outputs.branch }} - - # Setup Test Environment, build frontend but do not start server yet - - name: Setup ComfyUI server - uses: ./.github/actions/setup-comfyui-server - name: Setup frontend uses: ./.github/actions/setup-frontend with: include_build_step: true - - name: Setup Playwright - uses: ./.github/actions/setup-playwright - - # Save the entire workspace as cache for update jobs to restore + # Save expensive build artifacts (Python env, built frontend, node_modules) + # Source code will be checked out fresh in sharded jobs - name: Generate cache key id: cache-key run: echo "key=$(date +%s)" >> $GITHUB_OUTPUT - name: Save cache uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 with: - path: . + path: | + ComfyUI + dist + node_modules key: comfyui-setup-${{ steps.cache-key.outputs.key }} # Sharded snapshot updates @@ -91,21 +88,28 @@ jobs: shardIndex: [1, 2, 3, 4] shardTotal: [4] steps: - # Restore test environment from setup job - - name: Wait for cache propagation - run: sleep 10 + # Checkout source code fresh (not cached) + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ needs.setup.outputs.branch }} - - name: Restore cached setup + # Restore expensive build artifacts from setup job + - name: Restore cached artifacts uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 with: fail-on-cache-miss: true - path: . + path: | + ComfyUI + dist + node_modules key: comfyui-setup-${{ needs.setup.outputs.cache-key }} - - name: Setup ComfyUI server + - name: Setup ComfyUI server (from cache) uses: ./.github/actions/setup-comfyui-server with: launch_server: true + skip_checkout: true - name: Setup nodejs, pnpm, reuse built frontend uses: ./.github/actions/setup-frontend @@ -158,14 +162,44 @@ jobs: # Merge snapshots from all shards into browser_tests directory - name: Merge snapshots run: | + set -euo pipefail + echo "Merging snapshots from all shards..." + + # Verify target directory exists + if [ ! -d "browser_tests" ]; then + echo "::error::Target directory 'browser_tests' does not exist" + exit 1 + fi + + merged_count=0 for shard_dir in ./downloaded-snapshots/snapshots-shard-*; do - if [ -d "$shard_dir" ]; then - echo "Copying from $shard_dir..." - cp -r "$shard_dir"/* browser_tests/ 2>/dev/null || true + # Check if glob matched any directories + if [ ! -e "$shard_dir" ]; then + echo "::warning::No shard directories found matching pattern" + continue + fi + + if [ ! -d "$shard_dir" ]; then + echo "::warning::Expected directory but found file: $shard_dir" + continue fi + + # Check if directory has content (empty is OK - shard might have no changes) + if [ -z "$(ls -A "$shard_dir" 2>/dev/null)" ]; then + echo "Shard directory $shard_dir is empty (no snapshots updated in this shard)" + continue + fi + + echo "Copying from $shard_dir..." + if ! cp -r "$shard_dir"/* browser_tests/; then + echo "::error::Failed to copy snapshots from $shard_dir to browser_tests/" + exit 1 + fi + ((merged_count++)) done - echo "Snapshot merge complete" + + echo "Snapshot merge complete. Merged snapshots from $merged_count shard(s)" - name: Debugging info run: | diff --git a/src/components/graph/SelectionToolbox.vue b/src/components/graph/SelectionToolbox.vue index 06626ab1e1..587d7c9166 100644 --- a/src/components/graph/SelectionToolbox.vue +++ b/src/components/graph/SelectionToolbox.vue @@ -10,7 +10,7 @@ class="selection-toolbox pointer-events-auto rounded-lg border border-interface-stroke bg-interface-panel-surface" :pt="{ header: 'hidden', - content: 'p-2 h-12 flex flex-row gap-1' + content: 'p-2 h-2 flex flex-row gap-1' }" @wheel="canvasInteractions.forwardEventToCanvas" > From 28985a0e3b73834564942852bc3f24799ec145f7 Mon Sep 17 00:00:00 2001 From: Johnpaul Date: Tue, 21 Oct 2025 06:35:41 +0100 Subject: [PATCH 3/6] Add dispatch trigger for testing --- .github/workflows/pr-update-playwright-expectations.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-update-playwright-expectations.yaml b/.github/workflows/pr-update-playwright-expectations.yaml index 2e6d3df96a..13735d4a27 100644 --- a/.github/workflows/pr-update-playwright-expectations.yaml +++ b/.github/workflows/pr-update-playwright-expectations.yaml @@ -6,6 +6,7 @@ on: types: [labeled] issue_comment: types: [created] + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} From e9be12f41410161dda6f48b1b454a2b81965030c Mon Sep 17 00:00:00 2001 From: Johnpaul Date: Tue, 21 Oct 2025 08:42:32 +0100 Subject: [PATCH 4/6] fix expectations commiting --- .../pr-update-playwright-expectations.yaml | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-update-playwright-expectations.yaml b/.github/workflows/pr-update-playwright-expectations.yaml index 13735d4a27..05e464c062 100644 --- a/.github/workflows/pr-update-playwright-expectations.yaml +++ b/.github/workflows/pr-update-playwright-expectations.yaml @@ -173,34 +173,51 @@ jobs: exit 1 fi + # List all downloaded artifacts for debugging + echo "Available shard artifacts:" + ls -la ./downloaded-snapshots/ || echo "No downloaded-snapshots directory found" + merged_count=0 + skipped_count=0 + for shard_dir in ./downloaded-snapshots/snapshots-shard-*; do + echo "Processing: $shard_dir" + # Check if glob matched any directories if [ ! -e "$shard_dir" ]; then echo "::warning::No shard directories found matching pattern" - continue + break fi if [ ! -d "$shard_dir" ]; then echo "::warning::Expected directory but found file: $shard_dir" + skipped_count=$((skipped_count + 1)) continue fi # Check if directory has content (empty is OK - shard might have no changes) if [ -z "$(ls -A "$shard_dir" 2>/dev/null)" ]; then echo "Shard directory $shard_dir is empty (no snapshots updated in this shard)" + skipped_count=$((skipped_count + 1)) continue fi - echo "Copying from $shard_dir..." - if ! cp -r "$shard_dir"/* browser_tests/; then - echo "::error::Failed to copy snapshots from $shard_dir to browser_tests/" + echo "Merging snapshots from $shard_dir..." + + # The artifact path 'browser_tests/**/*-snapshots/' uploads relative to browser_tests/ + # So downloaded artifacts contain 'tests/' directly, not 'browser_tests/tests/' + # We need to merge the artifact contents into browser_tests/ + if ! rsync -a "$shard_dir/" browser_tests/; then + echo "::error::Failed to merge snapshots from $shard_dir/ to browser_tests/" + echo "Contents of $shard_dir:" + find "$shard_dir" -type f | head -20 exit 1 fi - ((merged_count++)) + merged_count=$((merged_count + 1)) + echo "Successfully merged shard $merged_count" done - echo "Snapshot merge complete. Merged snapshots from $merged_count shard(s)" + echo "Snapshot merge complete. Merged: $merged_count shard(s), Skipped: $skipped_count shard(s)" - name: Debugging info run: | From 401379e94ce792f33f1e47c3b9fbac0298c2c074 Mon Sep 17 00:00:00 2001 From: Johnpaul Date: Thu, 23 Oct 2025 06:08:15 +0100 Subject: [PATCH 5/6] remove unnesessary caching --- .github/actions/setup-comfyui-server/action.yml | 6 ------ .github/workflows/pr-update-playwright-expectations.yaml | 4 ---- 2 files changed, 10 deletions(-) diff --git a/.github/actions/setup-comfyui-server/action.yml b/.github/actions/setup-comfyui-server/action.yml index 7aaf9220bd..d1aa1bd578 100644 --- a/.github/actions/setup-comfyui-server/action.yml +++ b/.github/actions/setup-comfyui-server/action.yml @@ -9,10 +9,6 @@ inputs: description: 'Whether to launch the server after setup' required: false default: 'false' - skip_checkout: - description: 'Skip checking out ComfyUI repo (use when restoring from cache)' - required: false - default: 'false' runs: using: 'composite' steps: @@ -20,7 +16,6 @@ runs: # Checkout ComfyUI repo, install the dev_tools node and start server - name: Checkout ComfyUI - if: ${{ inputs.skip_checkout != 'true' }} uses: actions/checkout@v5 with: repository: 'comfyanonymous/ComfyUI' @@ -43,7 +38,6 @@ runs: python-version: '3.10' - name: Install Python requirements - if: ${{ inputs.skip_checkout != 'true' }} shell: bash working-directory: ComfyUI run: | diff --git a/.github/workflows/pr-update-playwright-expectations.yaml b/.github/workflows/pr-update-playwright-expectations.yaml index 05e464c062..8d82bd6757 100644 --- a/.github/workflows/pr-update-playwright-expectations.yaml +++ b/.github/workflows/pr-update-playwright-expectations.yaml @@ -6,7 +6,6 @@ on: types: [labeled] issue_comment: types: [created] - workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -76,7 +75,6 @@ jobs: path: | ComfyUI dist - node_modules key: comfyui-setup-${{ steps.cache-key.outputs.key }} # Sharded snapshot updates @@ -103,14 +101,12 @@ jobs: path: | ComfyUI dist - node_modules key: comfyui-setup-${{ needs.setup.outputs.cache-key }} - name: Setup ComfyUI server (from cache) uses: ./.github/actions/setup-comfyui-server with: launch_server: true - skip_checkout: true - name: Setup nodejs, pnpm, reuse built frontend uses: ./.github/actions/setup-frontend From 520443b0c4b0bd0cfddaaf0a14a71d5b0c79c6a0 Mon Sep 17 00:00:00 2001 From: Johnpaul Date: Thu, 23 Oct 2025 06:10:14 +0100 Subject: [PATCH 6/6] reset toolbox --- src/components/graph/SelectionToolbox.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/graph/SelectionToolbox.vue b/src/components/graph/SelectionToolbox.vue index 587d7c9166..06626ab1e1 100644 --- a/src/components/graph/SelectionToolbox.vue +++ b/src/components/graph/SelectionToolbox.vue @@ -10,7 +10,7 @@ class="selection-toolbox pointer-events-auto rounded-lg border border-interface-stroke bg-interface-panel-surface" :pt="{ header: 'hidden', - content: 'p-2 h-2 flex flex-row gap-1' + content: 'p-2 h-12 flex flex-row gap-1' }" @wheel="canvasInteractions.forwardEventToCanvas" >