diff --git a/.github/actions/start-comfyui-server/action.yml b/.github/actions/start-comfyui-server/action.yml new file mode 100644 index 000000000..2af727508 --- /dev/null +++ b/.github/actions/start-comfyui-server/action.yml @@ -0,0 +1,23 @@ +name: Start ComfyUI Server +description: 'Start ComfyUI server in a container environment (assumes ComfyUI is pre-installed)' + +inputs: + front_end_root: + description: 'Path to frontend dist directory' + required: false + default: '$GITHUB_WORKSPACE/dist' + timeout: + description: 'Timeout in seconds for server startup' + required: false + default: '600' + +runs: + using: 'composite' + steps: + - name: Copy devtools and start server + shell: bash + run: | + set -euo pipefail + cp -r ./tools/devtools/* /ComfyUI/custom_nodes/ComfyUI_devtools/ + cd /ComfyUI && python3 main.py --cpu --multi-user --front-end-root "${{ inputs.front_end_root }}" & + wait-for-it --service 127.0.0.1:8188 -t ${{ inputs.timeout }} diff --git a/.github/workflows/ci-tests-e2e.yaml b/.github/workflows/ci-tests-e2e.yaml index 5a2cc028b..12ccfae93 100644 --- a/.github/workflows/ci-tests-e2e.yaml +++ b/.github/workflows/ci-tests-e2e.yaml @@ -15,66 +15,56 @@ concurrency: jobs: setup: runs-on: ubuntu-latest - outputs: - cache-key: ${{ steps.cache-key.outputs.key }} steps: - name: Checkout repository uses: actions/checkout@v5 - - # Setup Test Environment, build frontend but do not start server yet - - name: Setup ComfyUI server - uses: ./.github/actions/setup-comfyui-server - name: Setup frontend uses: ./.github/actions/setup-frontend with: include_build_step: true - - name: Setup Playwright - uses: ./.github/actions/setup-playwright # Setup Playwright and cache browsers - # Save the entire workspace as cache for later test jobs to restore - - name: Generate cache key - id: cache-key - run: echo "key=$(date +%s)" >> $GITHUB_OUTPUT - - name: Save cache - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 + # Upload only built dist/ (containerized test jobs will pnpm install without cache) + - name: Upload built frontend + uses: actions/upload-artifact@v4 with: - path: . - key: comfyui-setup-${{ steps.cache-key.outputs.key }} + name: frontend-dist + path: dist/ + retention-days: 1 # Sharded chromium tests playwright-tests-chromium-sharded: needs: setup runs-on: ubuntu-latest timeout-minutes: 60 + container: + image: ghcr.io/comfy-org/comfyui-ci-container:0.0.8 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} permissions: contents: read + packages: read strategy: fail-fast: false matrix: shardIndex: [1, 2, 3, 4, 5, 6, 7, 8] shardTotal: [8] steps: - # download built frontend repo from setup job - - name: Wait for cache propagation - run: sleep 10 - - name: Restore cached setup - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 + - name: Checkout repository + uses: actions/checkout@v5 + - name: Download built frontend + uses: actions/download-artifact@v4 with: - fail-on-cache-miss: true - path: . - key: comfyui-setup-${{ needs.setup.outputs.cache-key }} + name: frontend-dist + path: dist/ - # Setup Test Environment for this runner, start server, use cached built frontend ./dist from 'setup' job - - name: Setup ComfyUI server - uses: ./.github/actions/setup-comfyui-server - with: - launch_server: true - - name: Setup nodejs, pnpm, reuse built frontend - uses: ./.github/actions/setup-frontend - - name: Setup Playwright - uses: ./.github/actions/setup-playwright + - name: Start ComfyUI server + uses: ./.github/actions/start-comfyui-server - # Run sharded tests and upload sharded reports + - name: Install frontend deps + run: pnpm install --frozen-lockfile + + # Run sharded tests (browsers pre-installed in container) - name: Run Playwright tests (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}) id: playwright run: pnpm exec playwright test --project=chromium --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --reporter=blob @@ -94,39 +84,37 @@ jobs: timeout-minutes: 15 needs: setup runs-on: ubuntu-latest + container: + image: ghcr.io/comfy-org/comfyui-ci-container:0.0.8 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} permissions: contents: read + packages: read strategy: fail-fast: false matrix: browser: [chromium-2x, chromium-0.5x, mobile-chrome] steps: - # download built frontend repo from setup job - - name: Wait for cache propagation - run: sleep 10 - - name: Restore cached setup - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 + - name: Checkout repository + uses: actions/checkout@v5 + - name: Download built frontend + uses: actions/download-artifact@v4 with: - fail-on-cache-miss: true - path: . - key: comfyui-setup-${{ needs.setup.outputs.cache-key }} + name: frontend-dist + path: dist/ - # Setup Test Environment for this runner, start server, use cached built frontend ./dist from 'setup' job - - name: Setup ComfyUI server - uses: ./.github/actions/setup-comfyui-server - with: - launch_server: true - - name: Setup nodejs, pnpm, reuse built frontend - uses: ./.github/actions/setup-frontend - - name: Setup Playwright - uses: ./.github/actions/setup-playwright + - name: Start ComfyUI server + uses: ./.github/actions/start-comfyui-server - # Run tests and upload reports + - name: Install frontend deps + run: pnpm install --frozen-lockfile + + # Run tests (browsers pre-installed in container) - name: Run Playwright tests (${{ matrix.browser }}) id: playwright - run: | - # Run tests with blob reporter first - pnpm exec playwright test --project=${{ matrix.browser }} --reporter=blob + run: pnpm exec playwright test --project=${{ matrix.browser }} --reporter=blob env: PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report @@ -147,7 +135,7 @@ jobs: path: ./playwright-report/ retention-days: 30 - # Merge sharded test reports + # Merge sharded test reports (no container needed - only runs CLI) merge-reports: needs: [playwright-tests-chromium-sharded] runs-on: ubuntu-latest @@ -156,11 +144,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 - # Setup Test Environment, we only need playwright to merge reports + # Setup pnpm/node to run playwright merge-reports (no browsers needed) - name: Setup frontend uses: ./.github/actions/setup-frontend - - name: Setup Playwright - uses: ./.github/actions/setup-playwright - name: Download blob reports uses: actions/download-artifact@v4 diff --git a/.github/workflows/pr-update-playwright-expectations.yaml b/.github/workflows/pr-update-playwright-expectations.yaml index d78ad96bf..e1c51c546 100644 --- a/.github/workflows/pr-update-playwright-expectations.yaml +++ b/.github/workflows/pr-update-playwright-expectations.yaml @@ -25,7 +25,6 @@ jobs: ) && startsWith(github.event.comment.body, '/update-playwright') ) outputs: - cache-key: ${{ steps.cache-key.outputs.key }} pr-number: ${{ steps.pr-info.outputs.pr-number }} branch: ${{ steps.pr-info.outputs.branch }} comment-id: ${{ steps.find-update-comment.outputs.comment-id }} @@ -64,70 +63,63 @@ jobs: uses: ./.github/actions/setup-frontend with: include_build_step: true - # Save expensive build artifacts (Python env, built frontend, node_modules) - # Source code will be checked out fresh in sharded jobs - - name: Generate cache key - id: cache-key - run: echo "key=$(date +%s)" >> $GITHUB_OUTPUT - - name: Save cache - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 + + # Upload built dist/ (containerized test jobs will pnpm install without cache) + - name: Upload built frontend + uses: actions/upload-artifact@v4 with: - path: | - ComfyUI - dist - key: comfyui-setup-${{ steps.cache-key.outputs.key }} + name: frontend-dist + path: dist/ + retention-days: 1 # Sharded snapshot updates update-snapshots-sharded: needs: setup runs-on: ubuntu-latest + container: + image: ghcr.io/comfy-org/comfyui-ci-container:0.0.8 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: read + packages: read strategy: fail-fast: false matrix: shardIndex: [1, 2, 3, 4] shardTotal: [4] steps: - # Checkout source code fresh (not cached) - name: Checkout repository uses: actions/checkout@v5 with: ref: ${{ needs.setup.outputs.branch }} - - # Restore expensive build artifacts from setup job - - name: Restore cached artifacts - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 + - name: Download built frontend + uses: actions/download-artifact@v4 with: - fail-on-cache-miss: true - path: | - ComfyUI - dist - key: comfyui-setup-${{ needs.setup.outputs.cache-key }} + name: frontend-dist + path: dist/ - - name: Setup ComfyUI server (from cache) - uses: ./.github/actions/setup-comfyui-server - with: - launch_server: true + - name: Start ComfyUI server + uses: ./.github/actions/start-comfyui-server - - name: Setup nodejs, pnpm, reuse built frontend - uses: ./.github/actions/setup-frontend + - name: Install frontend deps + run: pnpm install --frozen-lockfile - - name: Setup Playwright - uses: ./.github/actions/setup-playwright - - # Run sharded tests with snapshot updates + # Run sharded tests with snapshot updates (browsers pre-installed in container) - name: Update snapshots (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}) id: playwright-tests run: pnpm exec playwright test --update-snapshots --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} continue-on-error: true - # Identify and stage only changed snapshot files - name: Stage changed snapshot files id: changed-snapshots + shell: bash run: | set -euo pipefail - echo "==========================================" - echo "STAGING CHANGED SNAPSHOTS (Shard ${{ matrix.shardIndex }})" - echo "==========================================" + + # Configure git safe.directory for container environment + git config --global --add safe.directory "$(pwd)" # Get list of changed snapshot files (including untracked/new files) changed_files=$( ( @@ -136,43 +128,25 @@ jobs: ) | sort -u | grep -E '\-snapshots/' || true ) if [ -z "$changed_files" ]; then - echo "No snapshot changes in this shard" + echo "No snapshot changes in shard ${{ matrix.shardIndex }}" echo "has-changes=false" >> $GITHUB_OUTPUT exit 0 fi - echo "✓ Found changed files:" - echo "$changed_files" file_count=$(echo "$changed_files" | wc -l) - echo "Count: $file_count" + echo "Shard ${{ matrix.shardIndex }}: $file_count changed snapshot(s):" + echo "$changed_files" echo "has-changes=true" >> $GITHUB_OUTPUT - echo "" - # Create staging directory + # Copy changed files to staging directory mkdir -p /tmp/changed_snapshots_shard - - # Copy only changed files, preserving directory structure - # Strip 'browser_tests/' prefix to avoid double nesting - echo "Copying changed files to staging directory..." while IFS= read -r file; do - # Skip paths that no longer exist (e.g. deletions) - if [ ! -f "$file" ]; then - echo " → (skipped; not a file) $file" - continue - fi - # Remove 'browser_tests/' prefix + [ -f "$file" ] || continue file_without_prefix="${file#browser_tests/}" - # Create parent directories mkdir -p "/tmp/changed_snapshots_shard/$(dirname "$file_without_prefix")" - # Copy file cp "$file" "/tmp/changed_snapshots_shard/$file_without_prefix" - echo " → $file_without_prefix" done <<< "$changed_files" - echo "" - echo "Staged files for upload:" - find /tmp/changed_snapshots_shard -type f - # Upload ONLY the changed files from this shard - name: Upload changed snapshots uses: actions/upload-artifact@v4 @@ -213,9 +187,15 @@ jobs: echo "==========================================" echo "DOWNLOADED SNAPSHOT FILES" echo "==========================================" - find ./downloaded-snapshots -type f - echo "" - echo "Total files: $(find ./downloaded-snapshots -type f | wc -l)" + if [ -d "./downloaded-snapshots" ]; then + find ./downloaded-snapshots -type f + echo "" + echo "Total files: $(find ./downloaded-snapshots -type f | wc -l)" + else + echo "No snapshot artifacts downloaded (no changes in any shard)" + echo "" + echo "Total files: 0" + fi # Merge only the changed files into browser_tests - name: Merge changed snapshots @@ -226,6 +206,16 @@ jobs: echo "MERGING CHANGED SNAPSHOTS" echo "==========================================" + # Check if any artifacts were downloaded + if [ ! -d "./downloaded-snapshots" ]; then + echo "No snapshot artifacts to merge" + echo "==========================================" + echo "MERGE COMPLETE" + echo "==========================================" + echo "Shards merged: 0" + exit 0 + fi + # Verify target directory exists if [ ! -d "browser_tests" ]; then echo "::error::Target directory 'browser_tests' does not exist" diff --git a/browser_tests/tests/mobileBaseline.spec.ts-snapshots/mobile-settings-dialog-mobile-chrome-linux.png b/browser_tests/tests/mobileBaseline.spec.ts-snapshots/mobile-settings-dialog-mobile-chrome-linux.png index c179d1c75..207d037bf 100644 Binary files a/browser_tests/tests/mobileBaseline.spec.ts-snapshots/mobile-settings-dialog-mobile-chrome-linux.png and b/browser_tests/tests/mobileBaseline.spec.ts-snapshots/mobile-settings-dialog-mobile-chrome-linux.png differ diff --git a/browser_tests/tests/widget.spec.ts-snapshots/resized-to-original-chromium-linux.png b/browser_tests/tests/widget.spec.ts-snapshots/resized-to-original-chromium-linux.png index b67550204..2a35b4652 100644 Binary files a/browser_tests/tests/widget.spec.ts-snapshots/resized-to-original-chromium-linux.png and b/browser_tests/tests/widget.spec.ts-snapshots/resized-to-original-chromium-linux.png differ