ComfyUI_frontend/.github/workflows/test-ui.yaml

name: Tests CI

on:
  push:
    branches: [main, master, core/*, desktop/*]
  pull_request:
    branches-ignore:
      [wip/*, draft/*, temp/*, vue-nodes-migration, sno-playwright-*]

jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
      cache-key: ${{ steps.cache-key.outputs.key }}
      playwright-version: ${{ steps.playwright-version.outputs.PLAYWRIGHT_VERSION }}
    steps:
      - name: Checkout ComfyUI
        uses: actions/checkout@v4
        with:
          repository: 'comfyanonymous/ComfyUI'
          path: 'ComfyUI'
          ref: master

      - name: Checkout ComfyUI_frontend
        uses: actions/checkout@v4
        with:
          repository: 'Comfy-Org/ComfyUI_frontend'
          path: 'ComfyUI_frontend'

      - name: Checkout ComfyUI_devtools
        uses: actions/checkout@v4
        with:
          repository: 'Comfy-Org/ComfyUI_devtools'
          path: 'ComfyUI/custom_nodes/ComfyUI_devtools'
          ref: 'd05fd48dd787a4192e16802d4244cfcc0e2f9684'

      - name: Install pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 10

      - uses: actions/setup-node@v4
        with:
          node-version: lts/*
          cache: 'pnpm'
          cache-dependency-path: 'ComfyUI_frontend/pnpm-lock.yaml'

      - name: Cache tool outputs
        uses: actions/cache@v4
        with:
          path: |
            ComfyUI_frontend/.cache
            ComfyUI_frontend/tsconfig.tsbuildinfo
          key: playwright-setup-cache-${{ runner.os }}-${{ hashFiles('ComfyUI_frontend/pnpm-lock.yaml') }}-${{ hashFiles('ComfyUI_frontend/src/**/*.{ts,vue,js}', 'ComfyUI_frontend/*.config.*') }}
          restore-keys: |
            playwright-setup-cache-${{ runner.os }}-${{ hashFiles('ComfyUI_frontend/pnpm-lock.yaml') }}-
            playwright-setup-cache-${{ runner.os }}-
            playwright-tools-cache-${{ runner.os }}-

      - name: Build ComfyUI_frontend
        run: |
          pnpm install --frozen-lockfile
          pnpm build
        working-directory: ComfyUI_frontend

      - name: Generate cache key
        id: cache-key
        run: echo "key=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Playwright Version
        id: playwright-version
        run:  |
          PLAYWRIGHT_VERSION=$(pnpm ls @playwright/test --json | jq --raw-output '.[0].devDependencies["@playwright/test"].version')
          echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_OUTPUT
        working-directory: ComfyUI_frontend

      - name: Save cache
        uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684
        with:
          path: |
            ComfyUI
            ComfyUI_frontend
          key: comfyui-setup-${{ steps.cache-key.outputs.key }}

  # Sharded chromium tests
  playwright-tests-chromium-sharded:
    needs: setup
    runs-on: ubuntu-latest
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
        shardTotal: [8]
    steps:
      - name: Wait for cache propagation
        run: sleep 10

      - name: Restore cached setup
        uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684
        with:
          fail-on-cache-miss: true
          path: |
            ComfyUI
            ComfyUI_frontend
          key: comfyui-setup-${{ needs.setup.outputs.cache-key }}

      - name: Install pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 10

      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'
          cache: 'pip'

      - name: Install requirements
        run: |
          python -m pip install --upgrade pip
          pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
          pip install -r requirements.txt
          pip install wait-for-it
        working-directory: ComfyUI


      - name: Cache Playwright Browsers
        uses: actions/cache@v4
        id: cache-playwright-browsers
        with:
          path: '~/.cache/ms-playwright'
          key: '${{ runner.os }}-playwright-browsers-${{ needs.setup.outputs.playwright-version }}'

      - name: Install Playwright Browsers
        if: steps.cache-playwright-browsers.outputs.cache-hit != 'true'
        run: pnpm exec playwright install chromium --with-deps
        working-directory: ComfyUI_frontend

      - name: Install Playwright Browsers (operating system dependencies)
        if: steps.cache-playwright-browsers.outputs.cache-hit == 'true'
        run: pnpm exec playwright install-deps
        working-directory: ComfyUI_frontend

      - name: Start ComfyUI server
        run: |
          python main.py --cpu --multi-user --front-end-root ../ComfyUI_frontend/dist &
          wait-for-it --service 127.0.0.1:8188 -t 600
        working-directory: ComfyUI

      - name: Run Playwright tests (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
        id: playwright
        run: npx playwright test --project=chromium --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --reporter=blob
        working-directory: ComfyUI_frontend
        env:
          PLAYWRIGHT_BLOB_OUTPUT_DIR: ../blob-report

      - uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: blob-report-chromium-${{ matrix.shardIndex }}
          path: blob-report/
          retention-days: 1

  playwright-tests:
    # Ideally, each shard runs test in 6 minutes, but allow up to 15 minutes
    timeout-minutes: 15
    needs: setup
    runs-on: ubuntu-latest
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium-2x, chromium-0.5x, mobile-chrome]
    steps:
      - name: Wait for cache propagation
        run: sleep 10

      - name: Restore cached setup
        uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684
        with:
          fail-on-cache-miss: true
          path: |
            ComfyUI
            ComfyUI_frontend
          key: comfyui-setup-${{ needs.setup.outputs.cache-key }}

      - name: Install pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 10

      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'
          cache: 'pip'

      - name: Install requirements
        run: |
          python -m pip install --upgrade pip
          pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
          pip install -r requirements.txt
          pip install wait-for-it
        working-directory: ComfyUI

      - name: Cache Playwright Browsers
        uses: actions/cache@v4
        id: cache-playwright-browsers
        with:
          path: '~/.cache/ms-playwright'
          key: '${{ runner.os }}-playwright-browsers-${{ needs.setup.outputs.playwright-version }}'

      - name: Install Playwright Browsers
        if: steps.cache-playwright-browsers.outputs.cache-hit != 'true'
        run: pnpm exec playwright install chromium --with-deps
        working-directory: ComfyUI_frontend

      - name: Install Playwright Browsers (operating system dependencies)
        if: steps.cache-playwright-browsers.outputs.cache-hit == 'true'
        run: pnpm exec playwright install-deps
        working-directory: ComfyUI_frontend

      - name: Start ComfyUI server
        run: |
          python main.py --cpu --multi-user --front-end-root ../ComfyUI_frontend/dist &
          wait-for-it --service 127.0.0.1:8188 -t 600
        working-directory: ComfyUI

      - name: Run Playwright tests (${{ matrix.browser }})
        id: playwright
        run: |
          # Run tests with both HTML and JSON reporters
          PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
          npx playwright test --project=${{ matrix.browser }} \
            --reporter=list \
            --reporter=html \
            --reporter=json
        working-directory: ComfyUI_frontend

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: playwright-report-${{ matrix.browser }}
          path: ComfyUI_frontend/playwright-report/
          retention-days: 30

  # Merge sharded test reports
  merge-reports:
    needs: [playwright-tests-chromium-sharded]
    runs-on: ubuntu-latest
    if: ${{ !cancelled() }}
    steps:
      - name: Checkout ComfyUI_frontend
        uses: actions/checkout@v4
        with:
          repository: 'Comfy-Org/ComfyUI_frontend'
          path: 'ComfyUI_frontend'

      - name: Install pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 10

      - uses: actions/setup-node@v4
        with:
          node-version: lts/*
          cache: 'pnpm'
          cache-dependency-path: 'ComfyUI_frontend/pnpm-lock.yaml'

      - name: Install dependencies
        run: |
          pnpm install --frozen-lockfile
        working-directory: ComfyUI_frontend

      - name: Download blob reports
        uses: actions/download-artifact@v4
        with:
          path: ComfyUI_frontend/all-blob-reports
          pattern: blob-report-chromium-*
          merge-multiple: true

      - name: Merge into HTML Report
        run: |
          # Generate HTML report
          npx playwright merge-reports --reporter=html ./all-blob-reports
          # Generate JSON report separately with explicit output path
          PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
          npx playwright merge-reports --reporter=json ./all-blob-reports
        working-directory: ComfyUI_frontend

      - name: Upload HTML report
        uses: actions/upload-artifact@v4
        with:
          name: playwright-report-chromium
          path: ComfyUI_frontend/playwright-report/
          retention-days: 30

  #### BEGIN Deployment and commenting (non-forked PRs only)
  # when using pull_request event, we have permission to comment directly
  # if its a forked repo, we need to use workflow_run event in a separate workflow (pr-playwright-deploy.yaml)

  # Post starting comment for non-forked PRs
  comment-on-pr-start:
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
    permissions:
      pull-requests: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        
      - name: Get start time
        id: start-time
        run: echo "time=$(date -u '+%m/%d/%Y, %I:%M:%S %p')" >> $GITHUB_OUTPUT
        
      - name: Post starting comment
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          chmod +x scripts/cicd/pr-playwright-deploy-and-comment.sh
          ./scripts/cicd/pr-playwright-deploy-and-comment.sh \
            "${{ github.event.pull_request.number }}" \
            "${{ github.head_ref }}" \
            "starting" \
            "${{ steps.start-time.outputs.time }}"
          
  # Deploy and comment for non-forked PRs only
  deploy-and-comment:
    needs: [playwright-tests, merge-reports]
    runs-on: ubuntu-latest
    if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
    permissions:
      pull-requests: write
      contents: read
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        
      - name: Download all playwright reports
        uses: actions/download-artifact@v4
        with:
          pattern: playwright-report-*
          path: reports
          
      - name: Make deployment script executable
        run: chmod +x scripts/cicd/pr-playwright-deploy-and-comment.sh
        
      - name: Deploy reports and comment on PR
        env:
          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          ./scripts/cicd/pr-playwright-deploy-and-comment.sh \
            "${{ github.event.pull_request.number }}" \
            "${{ github.head_ref }}" \
            "completed"
  #### END Deployment and commenting (non-forked PRs only)