ComfyUI_frontend/.github/workflows/ci-website-e2e.yaml

name: 'CI: Website E2E'

on:
  push:
    branches: [main, website/*]
    paths:
      - 'apps/website/**'
      - 'packages/design-system/**'
      - 'packages/tailwind-utils/**'
      - 'pnpm-lock.yaml'
  pull_request:
    branches-ignore: [wip/*, draft/*, temp/*]
    paths:
      - 'apps/website/**'
      - 'packages/design-system/**'
      - 'packages/tailwind-utils/**'
      - 'pnpm-lock.yaml'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  website-e2e:
    runs-on: ubuntu-latest
    container:
      image: mcr.microsoft.com/playwright:v1.58.1-noble
    timeout-minutes: 15
    permissions:
      pull-requests: write
      contents: read
    steps:
      - uses: actions/checkout@v6

      - name: Post starting comment
        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
        uses: actions/github-script@v8
        with:
          script: |
            const marker = '<!-- WEBSITE_E2E_STATUS -->'
            const body = [
              marker,
              '## 🌐 Website E2E',
              '',
              '> [!NOTE]',
              `> Tests are running… [View workflow run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`
            ].join('\n')
            const { data: comments } = await github.rest.issues.listComments({
              ...context.repo,
              issue_number: context.issue.number
            })
            const existing = comments.find(c => c.body.includes(marker))
            if (existing) {
              await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body })
            } else {
              await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body })
            }

      - name: Install pnpm
        run: corepack enable && corepack prepare

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Build website
        run: pnpm --filter @comfyorg/website build

      - name: Run Playwright tests
        id: tests
        run: pnpm --filter @comfyorg/website test:e2e

      - name: Upload test report
        uses: actions/upload-artifact@v6
        if: ${{ !cancelled() }}
        with:
          name: website-playwright-report
          path: apps/website/playwright-report/
          retention-days: 30

      - name: Deploy report to Cloudflare
        id: deploy
        if: always() && !cancelled()
        env:
          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
          HEAD_REF: ${{ github.head_ref || github.ref_name }}
        run: |
          BRANCH=$(echo "$HEAD_REF" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g;s/--*/-/g;s/^-\|-$//g')
          DEPLOY_OK=false
          for i in 1 2 3; do
            echo "Deployment attempt $i of 3..."
            OUTPUT=$(npx wrangler@^4.0.0 pages deploy apps/website/playwright-report \
              --project-name=comfyui-website-e2e \
              --branch="$BRANCH" 2>&1) && { DEPLOY_OK=true; break; } || echo "$OUTPUT"
            [ $i -lt 3 ] && sleep 10
          done
          echo "$OUTPUT"
          if [ "$DEPLOY_OK" != "true" ]; then
            echo "::error::All 3 deployment attempts failed"
            exit 1
          fi
          URL=$(echo "$OUTPUT" | grep -oE 'https://[a-zA-Z0-9.-]+\.pages\.dev\S*' | head -1)
          echo "url=${URL}" >> $GITHUB_OUTPUT

      - name: Categorize failures
        id: failures
        if: always() && !cancelled() && steps.tests.outcome != 'success'
        uses: actions/github-script@v8
        with:
          script: |
            const fs = require('fs')
            const report = JSON.parse(fs.readFileSync('apps/website/results.json', 'utf8'))

            function isFailed(t) { return t.status === 'unexpected' || t.status === 'flaky' }
            function isVisual(spec) {
              return spec.file?.includes('visual') ||
                spec.tests?.some(t => t.results?.some(r => r.error?.message?.includes('toHaveScreenshot')))
            }
            function specsOf(suite) {
              return [
                ...(suite.specs || []),
                ...(suite.suites || []).flatMap(specsOf)
              ]
            }

            // True:  Visual
            // False: Other
            const failed = specsOf(report)
              .flatMap(spec => (spec.tests || [])
              .filter(isFailed)
              .map(() => isVisual(spec)))

            const screenshotFailures = failed.filter(Boolean).length
            core.setOutput('screenshot', screenshotFailures)
            core.setOutput('other', failed.length - screenshotFailures)

      - name: Post result comment and summary
        if: always() && !cancelled()
        uses: actions/github-script@v8
        with:
          script: |
            const passed = '${{ steps.tests.outcome }}' === 'success'
            const reportUrl = '${{ steps.deploy.outputs.url }}'
            const screenshotFailures = parseInt('${{ steps.failures.outputs.screenshot }}') || 0
            const otherFailures = parseInt('${{ steps.failures.outputs.other }}') || 0

            const lines = ['## 🌐 Website E2E', '']

            if (passed) {
              lines.push('> [!TIP]', '> All tests passed.')
            } else {
              lines.push('> [!CAUTION]', '> Some tests failed.')
            }

            // Results table
            const rows = [
              ['Status', passed ? '✅ Passed' : '❌ Failed'],
              ['Report', reportUrl ? `[View Report](${reportUrl})` : '_unavailable_']
            ]
            if (!passed) {
              rows.push(
                ['Screenshot diffs', String(screenshotFailures)],
                ['Other failures', String(otherFailures)]
              )
            }
            lines.push(
              '',
              '| | |',
              '|---|---|',
              ...rows.map(([k, v]) => `| **${k}** | ${v} |`)
            )

            if (screenshotFailures > 0) {
              const s = screenshotFailures === 1 ? '' : 's'
              lines.push('', `- [ ] Update website screenshots (${screenshotFailures} screenshot diff${s})`)
            }
            if (otherFailures > 0) {
              lines.push(
                '',
                '> [!WARNING]',
                `> ${otherFailures} non-screenshot failure${otherFailures === 1 ? '' : 's'} — these require manual review.`
              )
            }

            const summary = lines.join('\n')
            await core.summary.addRaw(summary).write()

            if (context.eventName === 'pull_request' && !context.payload.pull_request.head.repo.fork) {
              const marker = '<!-- WEBSITE_E2E_STATUS -->'
              const body = `${marker}\n${summary}`
              const { data: comments } = await github.rest.issues.listComments({
                ...context.repo,
                issue_number: context.issue.number
              })
              const existing = comments.find(c => c.body.includes(marker))
              if (existing) {
                await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body })
              } else {
                await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body })
              }
            }