CI: Use custom container for E2E tests (#7625)

## Summary
Use a pre-built container image with all dependencies for Playwright E2E
tests, eliminating ~130s setup time per shard.

## Changes
- Use `ghcr.io/comfy-org/comfyui-ci-container:0.0.8` container for test
jobs
- Container includes: Playwright browsers, Node.js, pnpm, Python,
ComfyUI backend (v0.5.1), all Python deps
- Simplified setup: just copy devtools and start server (no cloning, no
pip install, no browser setup)
- Add `version-bump*` to `branches-ignore` to skip E2E tests for version
bump PRs
- Replace cache with artifacts (cache doesn't work inside containers)

## Benefits
- ~130s faster per shard (no ComfyUI clone, no pip install, no browser
download)
- Consistent environment across all test jobs
- Simpler workflow configuration

## Container Image
Repository: https://github.com/comfy-org/comfyui-ci-container
Image: `ghcr.io/comfy-org/comfyui-ci-container:0.0.8`

## Test plan
- [x] Verify CI workflow runs with container
- [x] Verify Playwright tests pass
- [x] Verify snapshot updates work correctly

---------

Co-authored-by: github-actions <github-actions@github.com>

.github/actions/start-comfyui-server/action.yml

@@ -0,0 +1,23 @@
+name: Start ComfyUI Server
+description: 'Start ComfyUI server in a container environment (assumes ComfyUI is pre-installed)'
+
+inputs:
+  front_end_root:
+    description: 'Path to frontend dist directory'
+    required: false
+    default: '$GITHUB_WORKSPACE/dist'
+  timeout:
+    description: 'Timeout in seconds for server startup'
+    required: false
+    default: '600'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Copy devtools and start server
+      shell: bash
+      run: |
+        set -euo pipefail
+        cp -r ./tools/devtools/* /ComfyUI/custom_nodes/ComfyUI_devtools/
+        cd /ComfyUI && python3 main.py --cpu --multi-user --front-end-root "${{ inputs.front_end_root }}" &
+        wait-for-it --service 127.0.0.1:8188 -t ${{ inputs.timeout }}

.github/workflows/ci-tests-e2e.yaml

@@ -15,66 +15,56 @@ concurrency:
 jobs:
   setup:
     runs-on: ubuntu-latest
-    outputs:
-      cache-key: ${{ steps.cache-key.outputs.key }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v5
-
-      # Setup Test Environment, build frontend but do not start server yet
-      - name: Setup ComfyUI server
-        uses: ./.github/actions/setup-comfyui-server
       - name: Setup frontend
         uses: ./.github/actions/setup-frontend
         with:
           include_build_step: true
-      - name: Setup Playwright
-        uses: ./.github/actions/setup-playwright  # Setup Playwright and cache browsers
 
-      # Save the entire workspace as cache for later test jobs to restore
+      # Upload only built dist/ (containerized test jobs will pnpm install without cache)
-      - name: Generate cache key
+      - name: Upload built frontend
-        id: cache-key
+        uses: actions/upload-artifact@v4
-        run: echo "key=$(date +%s)" >> $GITHUB_OUTPUT
-      - name: Save cache
-        uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684
         with:
-          path: .
+          name: frontend-dist
-          key: comfyui-setup-${{ steps.cache-key.outputs.key }}
+          path: dist/
+          retention-days: 1
 
   # Sharded chromium tests
   playwright-tests-chromium-sharded:
     needs: setup
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/comfy-org/comfyui-ci-container:0.0.8
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
     permissions:
       contents: read
+      packages: read
     strategy:
       fail-fast: false
       matrix:
         shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
         shardTotal: [8]
     steps:
-      # download built frontend repo from setup job
+      - name: Checkout repository
-      - name: Wait for cache propagation
+        uses: actions/checkout@v5
-        run: sleep 10
+      - name: Download built frontend
-      - name: Restore cached setup
+        uses: actions/download-artifact@v4
-        uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684
         with:
-          fail-on-cache-miss: true
+          name: frontend-dist
-          path: .
+          path: dist/
-          key: comfyui-setup-${{ needs.setup.outputs.cache-key }}
 
-      # Setup Test Environment for this runner, start server, use cached built frontend ./dist from 'setup' job
+      - name: Start ComfyUI server
-      - name: Setup ComfyUI server
+        uses: ./.github/actions/start-comfyui-server
-        uses: ./.github/actions/setup-comfyui-server
-        with:
-          launch_server: true
-      - name: Setup nodejs, pnpm, reuse built frontend
-        uses: ./.github/actions/setup-frontend
-      - name: Setup Playwright
-        uses: ./.github/actions/setup-playwright
 
-      # Run sharded tests and upload sharded reports
+      - name: Install frontend deps
+        run: pnpm install --frozen-lockfile
+
+      # Run sharded tests (browsers pre-installed in container)
       - name: Run Playwright tests (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
         id: playwright
         run: pnpm exec playwright test --project=chromium --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --reporter=blob
@@ -94,39 +84,37 @@ jobs:
     timeout-minutes: 15
     needs: setup
     runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/comfy-org/comfyui-ci-container:0.0.8
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
     permissions:
       contents: read
+      packages: read
     strategy:
       fail-fast: false
       matrix:
         browser: [chromium-2x, chromium-0.5x, mobile-chrome]
     steps:
-      # download built frontend repo from setup job
+      - name: Checkout repository
-      - name: Wait for cache propagation
+        uses: actions/checkout@v5
-        run: sleep 10
+      - name: Download built frontend
-      - name: Restore cached setup
+        uses: actions/download-artifact@v4
-        uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684
         with:
-          fail-on-cache-miss: true
+          name: frontend-dist
-          path: .
+          path: dist/
-          key: comfyui-setup-${{ needs.setup.outputs.cache-key }}
 
-      # Setup Test Environment for this runner, start server, use cached built frontend ./dist from 'setup' job
+      - name: Start ComfyUI server
-      - name: Setup ComfyUI server
+        uses: ./.github/actions/start-comfyui-server
-        uses: ./.github/actions/setup-comfyui-server
-        with:
-          launch_server: true
-      - name: Setup nodejs, pnpm, reuse built frontend
-        uses: ./.github/actions/setup-frontend
-      - name: Setup Playwright
-        uses: ./.github/actions/setup-playwright
 
-      # Run tests and upload reports
+      - name: Install frontend deps
+        run: pnpm install --frozen-lockfile
+
+      # Run tests (browsers pre-installed in container)
       - name: Run Playwright tests (${{ matrix.browser }})
         id: playwright
-        run: |
+        run: pnpm exec playwright test --project=${{ matrix.browser }} --reporter=blob
-          # Run tests with blob reporter first
-          pnpm exec playwright test --project=${{ matrix.browser }} --reporter=blob
         env:
           PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report
 
@@ -147,7 +135,7 @@ jobs:
           path: ./playwright-report/
           retention-days: 30
 
-  # Merge sharded test reports
+  # Merge sharded test reports (no container needed - only runs CLI)
   merge-reports:
     needs: [playwright-tests-chromium-sharded]
     runs-on: ubuntu-latest
@@ -156,11 +144,9 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v5
 
-      # Setup Test Environment, we only need playwright to merge reports
+      # Setup pnpm/node to run playwright merge-reports (no browsers needed)
       - name: Setup frontend
         uses: ./.github/actions/setup-frontend
-      - name: Setup Playwright
-        uses: ./.github/actions/setup-playwright
 
       - name: Download blob reports
         uses: actions/download-artifact@v4

.github/workflows/pr-update-playwright-expectations.yaml

@@ -25,7 +25,6 @@ jobs:
         ) &&
         startsWith(github.event.comment.body, '/update-playwright') )
     outputs:
-      cache-key: ${{ steps.cache-key.outputs.key }}
       pr-number: ${{ steps.pr-info.outputs.pr-number }}
       branch: ${{ steps.pr-info.outputs.branch }}
       comment-id: ${{ steps.find-update-comment.outputs.comment-id }}
@@ -64,70 +63,63 @@ jobs:
         uses: ./.github/actions/setup-frontend
         with:
           include_build_step: true
-      # Save expensive build artifacts (Python env, built frontend, node_modules)
+
-      # Source code will be checked out fresh in sharded jobs
+      # Upload built dist/ (containerized test jobs will pnpm install without cache)
-      - name: Generate cache key
+      - name: Upload built frontend
-        id: cache-key
+        uses: actions/upload-artifact@v4
-        run: echo "key=$(date +%s)" >> $GITHUB_OUTPUT
-      - name: Save cache
-        uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684
         with:
-          path: |
+          name: frontend-dist
-            ComfyUI
+          path: dist/
-            dist
+          retention-days: 1
-          key: comfyui-setup-${{ steps.cache-key.outputs.key }}
 
   # Sharded snapshot updates
   update-snapshots-sharded:
     needs: setup
     runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/comfy-org/comfyui-ci-container:0.0.8
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    permissions:
+      contents: read
+      packages: read
     strategy:
       fail-fast: false
       matrix:
         shardIndex: [1, 2, 3, 4]
         shardTotal: [4]
     steps:
-      # Checkout source code fresh (not cached)
       - name: Checkout repository
         uses: actions/checkout@v5
         with:
           ref: ${{ needs.setup.outputs.branch }}
-
+      - name: Download built frontend
-      # Restore expensive build artifacts from setup job
+        uses: actions/download-artifact@v4
-      - name: Restore cached artifacts
-        uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684
         with:
-          fail-on-cache-miss: true
+          name: frontend-dist
-          path: |
+          path: dist/
-            ComfyUI
-            dist
-          key: comfyui-setup-${{ needs.setup.outputs.cache-key }}
 
-      - name: Setup ComfyUI server (from cache)
+      - name: Start ComfyUI server
-        uses: ./.github/actions/setup-comfyui-server
+        uses: ./.github/actions/start-comfyui-server
-        with:
-          launch_server: true
 
-      - name: Setup nodejs, pnpm, reuse built frontend
+      - name: Install frontend deps
-        uses: ./.github/actions/setup-frontend
+        run: pnpm install --frozen-lockfile
 
-      - name: Setup Playwright
+      # Run sharded tests with snapshot updates (browsers pre-installed in container)
-        uses: ./.github/actions/setup-playwright
-
-      # Run sharded tests with snapshot updates
       - name: Update snapshots (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
         id: playwright-tests
         run: pnpm exec playwright test --update-snapshots --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
         continue-on-error: true
 
-      # Identify and stage only changed snapshot files
       - name: Stage changed snapshot files
         id: changed-snapshots
+        shell: bash
         run: |
           set -euo pipefail
-          echo "=========================================="
+
-          echo "STAGING CHANGED SNAPSHOTS (Shard ${{ matrix.shardIndex }})"
+          # Configure git safe.directory for container environment
-          echo "=========================================="
+          git config --global --add safe.directory "$(pwd)"
 
           # Get list of changed snapshot files (including untracked/new files)
           changed_files=$( (
@@ -136,43 +128,25 @@ jobs:
           ) | sort -u | grep -E '\-snapshots/' || true )
 
           if [ -z "$changed_files" ]; then
-            echo "No snapshot changes in this shard"
+            echo "No snapshot changes in shard ${{ matrix.shardIndex }}"
             echo "has-changes=false" >> $GITHUB_OUTPUT
             exit 0
           fi
 
-          echo "✓ Found changed files:"
-          echo "$changed_files"
           file_count=$(echo "$changed_files" | wc -l)
-          echo "Count: $file_count"
+          echo "Shard ${{ matrix.shardIndex }}: $file_count changed snapshot(s):"
+          echo "$changed_files"
           echo "has-changes=true" >> $GITHUB_OUTPUT
-          echo ""
 
-          # Create staging directory
+          # Copy changed files to staging directory
           mkdir -p /tmp/changed_snapshots_shard
-
-          # Copy only changed files, preserving directory structure
-          # Strip 'browser_tests/' prefix to avoid double nesting
-          echo "Copying changed files to staging directory..."
           while IFS= read -r file; do
-            # Skip paths that no longer exist (e.g. deletions)
+            [ -f "$file" ] || continue
-            if [ ! -f "$file" ]; then
-              echo "  → (skipped; not a file) $file"
-              continue
-            fi
-            # Remove 'browser_tests/' prefix
             file_without_prefix="${file#browser_tests/}"
-            # Create parent directories
             mkdir -p "/tmp/changed_snapshots_shard/$(dirname "$file_without_prefix")"
-            # Copy file
             cp "$file" "/tmp/changed_snapshots_shard/$file_without_prefix"
-            echo "  → $file_without_prefix"
           done <<< "$changed_files"
 
-          echo ""
-          echo "Staged files for upload:"
-          find /tmp/changed_snapshots_shard -type f
-
       # Upload ONLY the changed files from this shard
       - name: Upload changed snapshots
         uses: actions/upload-artifact@v4
@@ -213,9 +187,15 @@ jobs:
           echo "=========================================="
           echo "DOWNLOADED SNAPSHOT FILES"
           echo "=========================================="
-          find ./downloaded-snapshots -type f
+          if [ -d "./downloaded-snapshots" ]; then
-          echo ""
+            find ./downloaded-snapshots -type f
-          echo "Total files: $(find ./downloaded-snapshots -type f | wc -l)"
+            echo ""
+            echo "Total files: $(find ./downloaded-snapshots -type f | wc -l)"
+          else
+            echo "No snapshot artifacts downloaded (no changes in any shard)"
+            echo ""
+            echo "Total files: 0"
+          fi
 
       # Merge only the changed files into browser_tests
       - name: Merge changed snapshots
@@ -226,6 +206,16 @@ jobs:
           echo "MERGING CHANGED SNAPSHOTS"
           echo "=========================================="
 
+          # Check if any artifacts were downloaded
+          if [ ! -d "./downloaded-snapshots" ]; then
+            echo "No snapshot artifacts to merge"
+            echo "=========================================="
+            echo "MERGE COMPLETE"
+            echo "=========================================="
+            echo "Shards merged: 0"
+            exit 0
+          fi
+
           # Verify target directory exists
           if [ ! -d "browser_tests" ]; then
             echo "::error::Target directory 'browser_tests' does not exist"