ComfyUI_frontend/.github/workflows/ci-tests-e2e.yaml

# Description: End-to-end testing with Playwright across multiple browsers, deploys test reports to Cloudflare Pages
name: 'CI: Tests E2E'

on:
  push:
    branches: [main, master, core/*, desktop/*]
  pull_request:
    branches-ignore:
      [wip/*, draft/*, temp/*, vue-nodes-migration, version-bump-*]
  # Run after i18n workflow completes for version-bump PRs
  workflow_run:
    workflows: ['i18n: Update Core']
    types: [completed]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.ref }}
  cancel-in-progress: true

jobs:
  setup:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v5
        with:
          ref: ${{ github.event.workflow_run.head_branch || github.ref }}
      - name: Setup frontend
        uses: ./.github/actions/setup-frontend
        with:
          include_build_step: true

      # Upload only built dist/ (containerized test jobs will pnpm install without cache)
      - name: Upload built frontend
        uses: actions/upload-artifact@v4
        with:
          name: frontend-dist
          path: dist/
          retention-days: 1

  # Sharded chromium tests
  playwright-tests-chromium-sharded:
    needs: setup
    runs-on: ubuntu-latest
    timeout-minutes: 60
    container:
      image: ghcr.io/comfy-org/comfyui-ci-container:0.0.10
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    permissions:
      contents: read
      packages: read
    strategy:
      fail-fast: false
      matrix:
        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
        shardTotal: [8]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v5
        with:
          ref: ${{ github.event.workflow_run.head_branch || github.ref }}
      - name: Download built frontend
        uses: actions/download-artifact@v4
        with:
          name: frontend-dist
          path: dist/

      - name: Start ComfyUI server
        uses: ./.github/actions/start-comfyui-server

      - name: Install frontend deps
        run: pnpm install --frozen-lockfile

      # Run sharded tests (browsers pre-installed in container)
      - name: Run Playwright tests (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
        id: playwright
        run: pnpm exec playwright test --project=chromium --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --reporter=blob
        env:
          PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report

      - name: Upload blob report
        uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: blob-report-chromium-${{ matrix.shardIndex }}
          path: blob-report/
          retention-days: 1

  playwright-tests:
    # Ideally, each shard runs test in 6 minutes, but allow up to 15 minutes
    timeout-minutes: 15
    needs: setup
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/comfy-org/comfyui-ci-container:0.0.10
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    permissions:
      contents: read
      packages: read
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium-2x, chromium-0.5x, mobile-chrome]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v5
        with:
          ref: ${{ github.event.workflow_run.head_branch || github.ref }}
      - name: Download built frontend
        uses: actions/download-artifact@v4
        with:
          name: frontend-dist
          path: dist/

      - name: Start ComfyUI server
        uses: ./.github/actions/start-comfyui-server

      - name: Install frontend deps
        run: pnpm install --frozen-lockfile

      # Run tests (browsers pre-installed in container)
      - name: Run Playwright tests (${{ matrix.browser }})
        id: playwright
        run: pnpm exec playwright test --project=${{ matrix.browser }} --reporter=blob
        env:
          PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report

      - name: Generate HTML and JSON reports
        if: always()
        run: |
          # Generate HTML report from blob
          pnpm exec playwright merge-reports --reporter=html ./blob-report
          # Generate JSON report separately with explicit output path
          PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
          pnpm exec playwright merge-reports --reporter=json ./blob-report

      - name: Upload Playwright report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: playwright-report-${{ matrix.browser }}
          path: ./playwright-report/
          retention-days: 30

  # Merge sharded test reports (no container needed - only runs CLI)
  merge-reports:
    needs: [playwright-tests-chromium-sharded]
    runs-on: ubuntu-latest
    if: ${{ !cancelled() }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v5
        with:
          ref: ${{ github.event.workflow_run.head_branch || github.ref }}

      - name: Install pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 10

      - name: Download blob reports
        uses: actions/download-artifact@v4
        with:
          path: ./all-blob-reports
          pattern: blob-report-chromium-*
          merge-multiple: true

      - name: Merge into HTML Report
        run: |
          # Generate HTML report
          pnpm dlx @playwright/test merge-reports --reporter=html ./all-blob-reports
          # Generate JSON report separately with explicit output path
          PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
          pnpm dlx @playwright/test merge-reports --reporter=json ./all-blob-reports

      - name: Upload HTML report
        uses: actions/upload-artifact@v4
        with:
          name: playwright-report-chromium
          path: ./playwright-report/
          retention-days: 30

  #### BEGIN Deployment and commenting (non-forked PRs only)
  # when using pull_request event, we have permission to comment directly
  # if its a forked repo, we need to use workflow_run event in a separate workflow (pr-playwright-deploy.yaml)

  # Get PR info once for reuse by comment jobs
  get-pr-info:
    runs-on: ubuntu-latest
    if: |
      (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) ||
      (github.event_name == 'workflow_run')
    outputs:
      pr_number: ${{ steps.get-pr.outputs.number }}
      branch: ${{ steps.get-pr.outputs.branch }}
    steps:
      - name: Get PR number
        id: get-pr
        env:
          GH_TOKEN: ${{ github.token }}
          PR_TARGET_REPO: ${{ github.repository }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
          PR_BRANCH: ${{ github.head_ref || github.event.workflow_run.head_branch }}
        run: |
          echo "branch=${PR_BRANCH}" >> $GITHUB_OUTPUT
          if [ -n "$PR_NUMBER" ]; then
            echo "number=${PR_NUMBER}" >> $GITHUB_OUTPUT
          else
            gh pr view --repo "${PR_TARGET_REPO}" "${PR_BRANCH}" \
              --json 'number' --jq '"number=\(.number)"' >> $GITHUB_OUTPUT
          fi

  # Post starting comment for non-forked PRs
  comment-on-pr-start:
    needs: get-pr-info
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v5
        with:
          ref: ${{ github.event.workflow_run.head_branch || github.ref }}

      - name: Get start time
        id: start-time
        run: echo "time=$(date -u '+%m/%d/%Y, %I:%M:%S %p')" >> $GITHUB_OUTPUT

      - name: Post starting comment
        env:
          GITHUB_TOKEN: ${{ github.token }}
          PR_NUMBER: ${{ needs.get-pr-info.outputs.pr_number }}
          BRANCH: ${{ needs.get-pr-info.outputs.branch }}
          START_TIME: ${{ steps.start-time.outputs.time }}
        run: |
          chmod +x scripts/cicd/pr-playwright-deploy-and-comment.sh
          ./scripts/cicd/pr-playwright-deploy-and-comment.sh \
            "$PR_NUMBER" \
            "$BRANCH" \
            "starting" \
            "$START_TIME"

  # Deploy and comment for non-forked PRs only
  deploy-and-comment:
    needs: [playwright-tests, merge-reports, get-pr-info]
    runs-on: ubuntu-latest
    if: always()
    permissions:
      pull-requests: write
      contents: read
    steps:
      - name: Checkout repository
        uses: actions/checkout@v5
        with:
          ref: ${{ github.event.workflow_run.head_branch || github.ref }}

      - name: Download all playwright reports
        uses: actions/download-artifact@v4
        with:
          pattern: playwright-report-*
          path: reports

      - name: Deploy reports and comment on PR
        env:
          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
          GITHUB_TOKEN: ${{ github.token }}
          GITHUB_SHA: ${{ github.event.pull_request.head.sha || github.event.workflow_run.head_sha }}
          PR_NUMBER: ${{ needs.get-pr-info.outputs.pr_number }}
          BRANCH: ${{ needs.get-pr-info.outputs.branch }}
        run: |
          bash ./scripts/cicd/pr-playwright-deploy-and-comment.sh \
            "$PR_NUMBER" \
            "$BRANCH" \
            "completed"
  #### END Deployment and commenting (non-forked PRs only)