mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-05-03 04:31:58 +00:00
8e3041aa2f
## Summary Follow-up to #11568. Fixes Playwright required checks hanging as "Waiting for status to be reported" on PRs with no e2e-relevant file changes. ## Problem PR #11568 added a `changes` filter to skip E2E when only docs/apps/storybook files are touched. The E2E workflow skips correctly, but branch rulesets require the 11 matrix-expanded check names (e.g. `playwright-tests-chromium-sharded (1, 8)`). When a matrix job is skipped via dependency, GitHub only reports the parent job name — the individual matrix entries are never reported, so required checks hang forever. ## Fix Add a single `e2e-status` gate job that: - Uses `if: always()` so it always runs regardless of skipped dependencies - Passes when E2E was intentionally skipped (no relevant changes) - Passes when all matrix jobs succeeded - Fails when any matrix job failed **After merging**, the ProtectMain and Core release branches rulesets should be updated to require `e2e-status` instead of the 11 individual matrix check names. ┆Issue is synchronized with this [Notion page](https://www.notion.so/PR-11587-ci-add-e2e-status-gate-job-for-required-checks-34c6d73d365081b59c01e8d61d0b808d) by [Unito](https://www.unito.io) Co-authored-by: Amp <amp@ampcode.com>
310 lines
10 KiB
YAML
310 lines
10 KiB
YAML
# Description: End-to-end testing with Playwright across multiple browsers, deploys test reports to Cloudflare Pages
|
|
name: 'CI: Tests E2E'
|
|
|
|
on:
|
|
push:
|
|
branches: [main, master, core/*, desktop/*]
|
|
paths-ignore: ['**/*.md']
|
|
pull_request:
|
|
branches-ignore: [wip/*, draft/*, temp/*]
|
|
merge_group:
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
# Detect whether e2e-relevant files changed. Required checks see "skipped"
|
|
# (which counts as passing) when only docs/apps/storybook files are touched,
|
|
# avoiding the stall that paths-ignore would cause.
|
|
changes:
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
should_run: ${{ github.event_name != 'pull_request' || steps.filter.outputs.e2e }}
|
|
steps:
|
|
- name: Checkout repository
|
|
if: ${{ github.event_name == 'pull_request' }}
|
|
uses: actions/checkout@v6
|
|
- name: Check for e2e-relevant changes
|
|
if: ${{ github.event_name == 'pull_request' }}
|
|
id: filter
|
|
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
|
|
with:
|
|
predicate-quantifier: 'every'
|
|
filters: |
|
|
e2e:
|
|
- '**'
|
|
- '!apps/**'
|
|
- '!docs/**'
|
|
- '!.storybook/**'
|
|
- '!**/*.md'
|
|
|
|
setup:
|
|
needs: changes
|
|
if: ${{ needs.changes.outputs.should_run == 'true' }}
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v6
|
|
- name: Setup frontend
|
|
uses: ./.github/actions/setup-frontend
|
|
with:
|
|
include_build_step: true
|
|
|
|
# Upload only built dist/ (containerized test jobs will pnpm install without cache)
|
|
- name: Upload built frontend
|
|
uses: actions/upload-artifact@v6
|
|
with:
|
|
name: frontend-dist
|
|
path: dist/
|
|
retention-days: 1
|
|
|
|
# Build cloud distribution for @cloud tagged tests
|
|
# NX_SKIP_NX_CACHE=true is required because `nx build` was already run
|
|
# for the OSS distribution above. Without skipping cache, Nx returns
|
|
# the cached OSS build since env vars aren't part of the cache key.
|
|
- name: Build cloud frontend
|
|
run: NX_SKIP_NX_CACHE=true pnpm build:cloud
|
|
|
|
- name: Upload cloud frontend
|
|
uses: actions/upload-artifact@v6
|
|
with:
|
|
name: frontend-dist-cloud
|
|
path: dist/
|
|
retention-days: 1
|
|
|
|
# Sharded chromium tests
|
|
playwright-tests-chromium-sharded:
|
|
needs: setup
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 60
|
|
container:
|
|
image: ghcr.io/comfy-org/comfyui-ci-container:0.0.16
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
permissions:
|
|
contents: read
|
|
packages: read
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
|
|
shardTotal: [8]
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v6
|
|
- name: Download built frontend
|
|
uses: actions/download-artifact@v7
|
|
with:
|
|
name: frontend-dist
|
|
path: dist/
|
|
|
|
- name: Start ComfyUI server
|
|
uses: ./.github/actions/start-comfyui-server
|
|
|
|
- name: Install frontend deps
|
|
run: pnpm install --frozen-lockfile
|
|
|
|
# Run sharded tests (browsers pre-installed in container)
|
|
- name: Run Playwright tests (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
|
|
id: playwright
|
|
run: pnpm exec playwright test --project=chromium --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --reporter=blob
|
|
env:
|
|
PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report
|
|
COLLECT_COVERAGE: 'true'
|
|
|
|
- name: Upload blob report
|
|
uses: actions/upload-artifact@v6
|
|
if: ${{ !cancelled() }}
|
|
with:
|
|
name: blob-report-chromium-${{ matrix.shardIndex }}
|
|
path: blob-report/
|
|
retention-days: 1
|
|
|
|
- name: Upload shard coverage data
|
|
if: always()
|
|
uses: actions/upload-artifact@v6
|
|
with:
|
|
name: e2e-coverage-shard-${{ matrix.shardIndex }}
|
|
path: coverage/playwright/
|
|
retention-days: 1
|
|
if-no-files-found: warn
|
|
|
|
playwright-tests:
|
|
# Ideally, each shard runs test in 6 minutes, but allow up to 15 minutes
|
|
timeout-minutes: 15
|
|
needs: setup
|
|
runs-on: ubuntu-latest
|
|
container:
|
|
image: ghcr.io/comfy-org/comfyui-ci-container:0.0.16
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
permissions:
|
|
contents: read
|
|
packages: read
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
browser: [chromium-2x, chromium-0.5x, mobile-chrome, cloud]
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v6
|
|
- name: Download built frontend
|
|
uses: actions/download-artifact@v7
|
|
with:
|
|
name: ${{ matrix.browser == 'cloud' && 'frontend-dist-cloud' || 'frontend-dist' }}
|
|
path: dist/
|
|
|
|
- name: Start ComfyUI server
|
|
uses: ./.github/actions/start-comfyui-server
|
|
|
|
- name: Install frontend deps
|
|
run: pnpm install --frozen-lockfile
|
|
|
|
# Run tests (browsers pre-installed in container)
|
|
- name: Run Playwright tests (${{ matrix.browser }})
|
|
id: playwright
|
|
run: pnpm exec playwright test --project=${{ matrix.browser }} --reporter=blob
|
|
env:
|
|
PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report
|
|
|
|
- name: Generate HTML and JSON reports
|
|
if: always()
|
|
run: |
|
|
# Generate HTML report from blob
|
|
pnpm exec playwright merge-reports --reporter=html ./blob-report
|
|
# Generate JSON report separately with explicit output path
|
|
PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
|
|
pnpm exec playwright merge-reports --reporter=json ./blob-report
|
|
|
|
- name: Upload Playwright report
|
|
uses: actions/upload-artifact@v6
|
|
if: always()
|
|
with:
|
|
name: playwright-report-${{ matrix.browser }}
|
|
path: ./playwright-report/
|
|
retention-days: 30
|
|
|
|
# Merge sharded test reports (no container needed - only runs CLI)
|
|
merge-reports:
|
|
needs: [changes, playwright-tests-chromium-sharded]
|
|
runs-on: ubuntu-latest
|
|
if: ${{ !cancelled() && needs.changes.outputs.should_run == 'true' }}
|
|
steps:
|
|
- name: Install pnpm
|
|
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0
|
|
with:
|
|
version: 10
|
|
|
|
- name: Download blob reports
|
|
uses: actions/download-artifact@v7
|
|
with:
|
|
path: ./all-blob-reports
|
|
pattern: blob-report-chromium-*
|
|
merge-multiple: true
|
|
|
|
- name: Merge into HTML Report
|
|
run: |
|
|
# Generate HTML report
|
|
pnpm dlx @playwright/test merge-reports --reporter=html ./all-blob-reports
|
|
# Generate JSON report separately with explicit output path
|
|
PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
|
|
pnpm dlx @playwright/test merge-reports --reporter=json ./all-blob-reports
|
|
|
|
- name: Upload HTML report
|
|
uses: actions/upload-artifact@v6
|
|
with:
|
|
name: playwright-report-chromium
|
|
path: ./playwright-report/
|
|
retention-days: 30
|
|
|
|
# Gate job — single required check that passes whether the matrix ran or was
|
|
# skipped. Branch rulesets require this instead of the individual matrix-
|
|
# expanded check names so PRs with no e2e-relevant changes aren't stuck.
|
|
e2e-status:
|
|
if: ${{ always() }}
|
|
needs: [changes, playwright-tests-chromium-sharded, playwright-tests]
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Check E2E results
|
|
env:
|
|
SHOULD_RUN: ${{ needs.changes.outputs.should_run }}
|
|
SHARDED: ${{ needs.playwright-tests-chromium-sharded.result }}
|
|
BROWSERS: ${{ needs.playwright-tests.result }}
|
|
run: |
|
|
[[ "$SHOULD_RUN" != "true" ]] && echo "E2E skipped" && exit 0
|
|
[[ "$SHARDED" != "success" || "$BROWSERS" != "success" ]] && echo "E2E failed" && exit 1
|
|
echo "E2E passed"
|
|
|
|
#### BEGIN Deployment and commenting (non-forked PRs only)
|
|
# when using pull_request event, we have permission to comment directly
|
|
# if its a forked repo, we need to use workflow_run event in a separate workflow (pr-playwright-deploy.yaml)
|
|
|
|
# Post starting comment for non-forked PRs
|
|
comment-on-pr-start:
|
|
needs: changes
|
|
runs-on: ubuntu-latest
|
|
if: >-
|
|
${{
|
|
needs.changes.outputs.should_run == 'true' &&
|
|
github.event_name == 'pull_request' &&
|
|
github.event.pull_request.head.repo.fork == false
|
|
}}
|
|
permissions:
|
|
pull-requests: write
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Post starting comment
|
|
env:
|
|
GITHUB_TOKEN: ${{ github.token }}
|
|
run: |
|
|
chmod +x scripts/cicd/pr-playwright-deploy-and-comment.sh
|
|
./scripts/cicd/pr-playwright-deploy-and-comment.sh \
|
|
"${{ github.event.pull_request.number }}" \
|
|
"${{ github.head_ref }}" \
|
|
"starting"
|
|
|
|
# Deploy and comment for non-forked PRs only
|
|
deploy-and-comment:
|
|
needs: [changes, playwright-tests, merge-reports]
|
|
runs-on: ubuntu-latest
|
|
if: >-
|
|
${{
|
|
always() &&
|
|
needs.changes.outputs.should_run == 'true' &&
|
|
github.event_name == 'pull_request' &&
|
|
github.event.pull_request.head.repo.fork == false
|
|
}}
|
|
permissions:
|
|
pull-requests: write
|
|
contents: read
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Download all playwright reports
|
|
uses: actions/download-artifact@v7
|
|
with:
|
|
pattern: playwright-report-*
|
|
path: reports
|
|
|
|
- name: Deploy reports and comment on PR
|
|
env:
|
|
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
|
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
|
GITHUB_TOKEN: ${{ github.token }}
|
|
GITHUB_SHA: ${{ github.event.pull_request.head.sha }}
|
|
run: |
|
|
bash ./scripts/cicd/pr-playwright-deploy-and-comment.sh \
|
|
"${{ github.event.pull_request.number }}" \
|
|
"${{ github.head_ref }}" \
|
|
"completed"
|
|
#### END Deployment and commenting (non-forked PRs only)
|