Only update snapshots of failed

.github/workflows/tests-ci.yaml

@@ -212,7 +212,7 @@ jobs:
   merge-reports:
     needs: [playwright-tests-chromium-sharded]
     runs-on: ubuntu-latest
-    if: ${{ !cancelled() }}
+    if: ${{ always() && !cancelled() }}
     steps:
       - name: Checkout ComfyUI_frontend
         uses: actions/checkout@v5
@@ -252,6 +252,22 @@ jobs:
           pnpm exec playwright merge-reports --reporter=json ./all-blob-reports
         working-directory: ComfyUI_frontend
 
+      - name: Build failed screenshot manifest
+        if: ${{ needs.playwright-tests-chromium-sharded.result == 'failure' }}
+        run: |
+          set -euo pipefail
+          pnpm tsx scripts/cicd/build-failed-screenshot-manifest.ts
+        working-directory: ComfyUI_frontend
+
+      - name: Upload failed screenshot manifest
+        if: ${{ needs.playwright-tests-chromium-sharded.result == 'failure' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: failed-screenshot-tests
+          path: ComfyUI_frontend/ci-rerun/*.txt
+          retention-days: 7
+          if-no-files-found: ignore
+
       - name: Upload HTML report
         uses: actions/upload-artifact@v4
         with:
@@ -319,4 +335,4 @@ jobs:
             "${{ github.event.pull_request.number }}" \
             "${{ github.head_ref }}" \
             "completed"
-  #### END Deployment and commenting (non-forked PRs only)
+  #### END Deployment and commenting (non-forked PRs only)

.github/workflows/update-playwright-expectations.yaml

@@ -16,11 +16,79 @@ jobs:
       uses: ./.github/actions/setup-frontend
     - name: Setup Playwright
       uses: ./.github/actions/setup-playwright
-    - name: Run Playwright tests and update snapshots
+    - name: Locate failed screenshot manifest artifact
+      id: locate-manifest
+      uses: actions/github-script@v8
+      with:
+        script: |
+          const { owner, repo } = context.repo
+          const headSha = context.payload.pull_request.head.sha
+
+          const { data } = await github.rest.actions.listWorkflowRuns({
+            owner,
+            repo,
+            workflow_id: 'tests-ci.yaml',
+            head_sha: headSha,
+            event: 'pull_request',
+            per_page: 1,
+          })
+          const run = data.workflow_runs?.[0]
+
+          let has = 'false'
+          let runId = ''
+          if (run) {
+            runId = String(run.id)
+            const { data: { artifacts = [] } } = await github.rest.actions.listWorkflowRunArtifacts({
+              owner,
+              repo,
+              run_id: run.id,
+              per_page: 100,
+            })
+            if (artifacts.some(a => a.name === 'failed-screenshot-tests' && !a.expired)) has = 'true'
+          }
+          core.setOutput('run_id', runId)
+          core.setOutput('has_manifest', has)
+
+    - name: Download failed screenshot manifest
+      if: steps.locate-manifest.outputs.has_manifest == 'true'
+      uses: actions/download-artifact@v4
+      with:
+        run-id: ${{ steps.locate-manifest.outputs.run_id }}
+        name: failed-screenshot-tests
+        path: ComfyUI_frontend/ci-rerun
+
+    - name: Re-run failed screenshot tests and update snapshots
       id: playwright-tests
-      run: pnpm exec playwright test --update-snapshots
-      continue-on-error: true
+      shell: bash
       working-directory: ComfyUI_frontend
+      run: |
+        set -euo pipefail
+        if [ ! -d ci-rerun ]; then
+          echo "No manifest found; running full suite as fallback"
+          pnpm exec playwright test --update-snapshots
+          exit 0
+        fi
+        shopt -s nullglob
+        files=(ci-rerun/*.txt)
+        if [ ${#files[@]} -eq 0 ]; then
+          echo "Manifest is empty; running full suite as fallback"
+          pnpm exec playwright test --update-snapshots
+          exit 0
+        fi
+        for f in "${files[@]}"; do
+          project="$(basename "$f" .txt)"
+          mapfile -t lines < "$f"
+          # Filter out blank lines
+          filtered=( )
+          for l in "${lines[@]}"; do
+            [ -n "$l" ] && filtered+=("$l")
+          done
+          if [ ${#filtered[@]} -eq 0 ]; then
+            continue
+          fi
+          echo "Re-running ${#filtered[@]} tests for project $project"
+          pnpm exec playwright test --project="$project" --update-snapshots "${filtered[@]}"
+        done
     - uses: actions/upload-artifact@v4
       if: always()
       with:
@@ -28,17 +96,21 @@ jobs:
         path: ComfyUI_frontend/playwright-report/
         retention-days: 30
     - name: Debugging info
+      working-directory: ComfyUI_frontend
       run: |
         echo "Branch: ${{ github.head_ref }}"
         git status
-      working-directory: ComfyUI_frontend
     - name: Commit updated expectations
+      working-directory: ComfyUI_frontend
       run: |
         git config --global user.name 'github-actions'
         git config --global user.email 'github-actions@github.com'
         git fetch origin ${{ github.head_ref }}
         git checkout -B ${{ github.head_ref }} origin/${{ github.head_ref }}
         git add browser_tests
-        git commit -m "[automated] Update test expectations"
-        git push origin HEAD:${{ github.head_ref }}
-      working-directory: ComfyUI_frontend
+        if git diff --cached --quiet; then
+          echo "No expectation updates detected; skipping commit."
+        else
+          git commit -m "[automated] Update test expectations"
+          git push origin HEAD:${{ github.head_ref }}
+        fi

scripts/cicd/build-failed-screenshot-manifest.ts

@@ -0,0 +1,74 @@
+import type {
+  JSONReport,
+  JSONReportSpec,
+  JSONReportSuite,
+  JSONReportTestResult
+} from '@playwright/test/reporter'
+import fs from 'node:fs'
+import fsp from 'node:fs/promises'
+import path from 'node:path'
+
+const argv = process.argv.slice(2)
+const getArg = (flag: string, fallback: string) => {
+  const i = argv.indexOf(flag)
+  if (i >= 0 && i + 1 < argv.length) return argv[i + 1]
+  return fallback
+}
+
+async function main() {
+  // Defaults mirror the workflow layout
+  const reportPath = getArg(
+    '--report',
+    path.join('playwright-report', 'report.json')
+  )
+  const outDir = getArg('--out', path.join('ci-rerun'))
+
+  if (!fs.existsSync(reportPath)) {
+    throw Error(`Report not found at ${reportPath}`)
+  }
+
+  const raw = await fsp.readFile(reportPath, 'utf8')
+  const data = JSON.parse(raw)
+
+  const hasScreenshotSignal = (r: JSONReportTestResult) => {
+    return r.attachments.some((att) => att?.contentType?.startsWith('image/'))
+  }
+
+  const out = new Map<string, Set<string>>()
+
+  const collectFailedScreenshots = (suite?: JSONReportSuite) => {
+    if (!suite) return
+    const childSuites = suite.suites ?? []
+    for (const childSuite of childSuites) collectFailedScreenshots(childSuite)
+    const specs: JSONReportSpec[] = suite.specs ?? []
+    for (const spec of specs) {
+      const file = spec.file
+      const line = spec.line
+      const loc = `${file}:${line}`
+      for (const test of spec.tests) {
+        const project = test.projectId
+        const last = test.results[test.results.length - 1]
+        const failedScreenshot =
+          last && last.status === 'failed' && hasScreenshotSignal(last)
+        if (!failedScreenshot) continue
+        if (!out.has(project)) out.set(project, new Set())
+        out.get(project)!.add(loc)
+      }
+    }
+  }
+
+  const report: JSONReport = data
+  const rootSuites = report.suites ?? []
+  for (const suite of rootSuites) collectFailedScreenshots(suite)
+
+  await fsp.mkdir(outDir, { recursive: true })
+  for (const [project, set] of out.entries()) {
+    const f = path.join(outDir, `${project}.txt`)
+    await fsp.writeFile(f, Array.from(set).join('\n') + '\n', 'utf8')
+  }
+}
+
+main().catch((err) => {
+  console.error('Manifest generation failed:', err)
+  process.exit(1)
+})