mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-03-02 19:49:58 +00:00
5e55f13e87
The sharded chromium tests were generating deployment info only from shard 1, which could miss failures from other shards. This change moves the deployment info generation to the merge-reports job where we have visibility into the overall result of all shards. Fixes #5338
335 lines
11 KiB
YAML
335 lines
11 KiB
YAML
name: Tests CI
|
|
|
|
on:
|
|
push:
|
|
branches: [main, master, core/*, desktop/*]
|
|
pull_request:
|
|
branches-ignore:
|
|
[wip/*, draft/*, temp/*, vue-nodes-migration, sno-playwright-*]
|
|
|
|
jobs:
|
|
setup:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
cache-key: ${{ steps.cache-key.outputs.key }}
|
|
playwright-version: ${{ steps.playwright-version.outputs.PLAYWRIGHT_VERSION }}
|
|
steps:
|
|
- name: Checkout ComfyUI
|
|
uses: actions/checkout@v4
|
|
with:
|
|
repository: 'comfyanonymous/ComfyUI'
|
|
path: 'ComfyUI'
|
|
ref: master
|
|
|
|
- name: Checkout ComfyUI_frontend
|
|
uses: actions/checkout@v4
|
|
with:
|
|
repository: 'Comfy-Org/ComfyUI_frontend'
|
|
path: 'ComfyUI_frontend'
|
|
|
|
- name: Checkout ComfyUI_devtools
|
|
uses: actions/checkout@v4
|
|
with:
|
|
repository: 'Comfy-Org/ComfyUI_devtools'
|
|
path: 'ComfyUI/custom_nodes/ComfyUI_devtools'
|
|
ref: 'd05fd48dd787a4192e16802d4244cfcc0e2f9684'
|
|
|
|
- name: Install pnpm
|
|
uses: pnpm/action-setup@v4
|
|
with:
|
|
version: 10
|
|
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: lts/*
|
|
cache: 'pnpm'
|
|
cache-dependency-path: 'ComfyUI_frontend/pnpm-lock.yaml'
|
|
|
|
- name: Cache tool outputs
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
ComfyUI_frontend/.cache
|
|
ComfyUI_frontend/tsconfig.tsbuildinfo
|
|
key: playwright-setup-cache-${{ runner.os }}-${{ hashFiles('ComfyUI_frontend/pnpm-lock.yaml') }}-${{ hashFiles('ComfyUI_frontend/src/**/*.{ts,vue,js}', 'ComfyUI_frontend/*.config.*') }}
|
|
restore-keys: |
|
|
playwright-setup-cache-${{ runner.os }}-${{ hashFiles('ComfyUI_frontend/pnpm-lock.yaml') }}-
|
|
playwright-setup-cache-${{ runner.os }}-
|
|
playwright-tools-cache-${{ runner.os }}-
|
|
|
|
- name: Build ComfyUI_frontend
|
|
run: |
|
|
pnpm install --frozen-lockfile
|
|
pnpm build
|
|
working-directory: ComfyUI_frontend
|
|
|
|
- name: Generate cache key
|
|
id: cache-key
|
|
run: echo "key=$(date +%s)" >> $GITHUB_OUTPUT
|
|
|
|
- name: Playwright Version
|
|
id: playwright-version
|
|
run: |
|
|
PLAYWRIGHT_VERSION=$(pnpm ls @playwright/test --json | jq --raw-output '.[0].devDependencies["@playwright/test"].version')
|
|
echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_OUTPUT
|
|
working-directory: ComfyUI_frontend
|
|
|
|
- name: Save cache
|
|
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684
|
|
with:
|
|
path: |
|
|
ComfyUI
|
|
ComfyUI_frontend
|
|
key: comfyui-setup-${{ steps.cache-key.outputs.key }}
|
|
|
|
# Sharded chromium tests
|
|
playwright-tests-chromium-sharded:
|
|
needs: setup
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
|
|
shardTotal: [8]
|
|
steps:
|
|
- name: Wait for cache propagation
|
|
run: sleep 10
|
|
|
|
- name: Restore cached setup
|
|
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684
|
|
with:
|
|
fail-on-cache-miss: true
|
|
path: |
|
|
ComfyUI
|
|
ComfyUI_frontend
|
|
key: comfyui-setup-${{ needs.setup.outputs.cache-key }}
|
|
|
|
- name: Install pnpm
|
|
uses: pnpm/action-setup@v4
|
|
with:
|
|
version: 10
|
|
|
|
- uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '3.10'
|
|
cache: 'pip'
|
|
|
|
- name: Install requirements
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
|
|
pip install -r requirements.txt
|
|
pip install wait-for-it
|
|
working-directory: ComfyUI
|
|
|
|
|
|
- name: Cache Playwright Browsers
|
|
uses: actions/cache@v4
|
|
id: cache-playwright-browsers
|
|
with:
|
|
path: '~/.cache/ms-playwright'
|
|
key: '${{ runner.os }}-playwright-browsers-${{ needs.setup.outputs.playwright-version }}'
|
|
|
|
- name: Install Playwright Browsers
|
|
if: steps.cache-playwright-browsers.outputs.cache-hit != 'true'
|
|
run: pnpm exec playwright install chromium --with-deps
|
|
working-directory: ComfyUI_frontend
|
|
|
|
- name: Install Playwright Browsers (operating system dependencies)
|
|
if: steps.cache-playwright-browsers.outputs.cache-hit == 'true'
|
|
run: pnpm exec playwright install-deps
|
|
working-directory: ComfyUI_frontend
|
|
|
|
- name: Start ComfyUI server
|
|
run: |
|
|
python main.py --cpu --multi-user --front-end-root ../ComfyUI_frontend/dist &
|
|
wait-for-it --service 127.0.0.1:8188 -t 600
|
|
working-directory: ComfyUI
|
|
|
|
- name: Run Playwright tests (Shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
|
|
id: playwright
|
|
run: npx playwright test --project=chromium --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --reporter=blob
|
|
working-directory: ComfyUI_frontend
|
|
env:
|
|
PLAYWRIGHT_BLOB_OUTPUT_DIR: ../blob-report
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
if: ${{ !cancelled() }}
|
|
with:
|
|
name: blob-report-chromium-${{ matrix.shardIndex }}
|
|
path: blob-report/
|
|
retention-days: 1
|
|
|
|
playwright-tests:
|
|
# Ideally, each shard runs test in 6 minutes, but allow up to 15 minutes
|
|
timeout-minutes: 15
|
|
needs: setup
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
browser: [chromium-2x, chromium-0.5x, mobile-chrome]
|
|
steps:
|
|
- name: Wait for cache propagation
|
|
run: sleep 10
|
|
|
|
- name: Restore cached setup
|
|
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684
|
|
with:
|
|
fail-on-cache-miss: true
|
|
path: |
|
|
ComfyUI
|
|
ComfyUI_frontend
|
|
key: comfyui-setup-${{ needs.setup.outputs.cache-key }}
|
|
|
|
- name: Install pnpm
|
|
uses: pnpm/action-setup@v4
|
|
with:
|
|
version: 10
|
|
|
|
- uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '3.10'
|
|
cache: 'pip'
|
|
|
|
- name: Install requirements
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
|
|
pip install -r requirements.txt
|
|
pip install wait-for-it
|
|
working-directory: ComfyUI
|
|
|
|
- name: Cache Playwright Browsers
|
|
uses: actions/cache@v4
|
|
id: cache-playwright-browsers
|
|
with:
|
|
path: '~/.cache/ms-playwright'
|
|
key: '${{ runner.os }}-playwright-browsers-${{ needs.setup.outputs.playwright-version }}'
|
|
|
|
- name: Install Playwright Browsers
|
|
if: steps.cache-playwright-browsers.outputs.cache-hit != 'true'
|
|
run: pnpm exec playwright install chromium --with-deps
|
|
working-directory: ComfyUI_frontend
|
|
|
|
- name: Install Playwright Browsers (operating system dependencies)
|
|
if: steps.cache-playwright-browsers.outputs.cache-hit == 'true'
|
|
run: pnpm exec playwright install-deps
|
|
working-directory: ComfyUI_frontend
|
|
|
|
- name: Start ComfyUI server
|
|
run: |
|
|
python main.py --cpu --multi-user --front-end-root ../ComfyUI_frontend/dist &
|
|
wait-for-it --service 127.0.0.1:8188 -t 600
|
|
working-directory: ComfyUI
|
|
|
|
- name: Run Playwright tests (${{ matrix.browser }})
|
|
id: playwright
|
|
run: npx playwright test --project=${{ matrix.browser }} --reporter=html
|
|
working-directory: ComfyUI_frontend
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: playwright-report-${{ matrix.browser }}
|
|
path: ComfyUI_frontend/playwright-report/
|
|
retention-days: 30
|
|
|
|
# Save test result for deployment info
|
|
- name: Save test result for ${{ matrix.browser }}
|
|
if: always()
|
|
run: |
|
|
# Use outcome to determine exit code (0 for success, 1 for failure/skipped)
|
|
if [ "${{ steps.playwright.outcome }}" = "success" ]; then
|
|
EXIT_CODE=0
|
|
else
|
|
EXIT_CODE=1
|
|
fi
|
|
# Generate project name based on browser
|
|
if [ "${{ matrix.browser }}" = "chromium-0.5x" ]; then
|
|
PROJECT_NAME="comfyui-playwright-chromium-0-5x"
|
|
else
|
|
PROJECT_NAME="comfyui-playwright-${{ matrix.browser }}"
|
|
fi
|
|
echo "${{ matrix.browser }}|${EXIT_CODE}|https://${PROJECT_NAME}.pages.dev" > deployment-info-${{ matrix.browser }}.txt
|
|
working-directory: ComfyUI_frontend
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: deployment-info-${{ matrix.browser }}
|
|
path: ComfyUI_frontend/deployment-info-${{ matrix.browser }}.txt
|
|
retention-days: 1
|
|
|
|
# Merge sharded test reports
|
|
merge-reports:
|
|
needs: [playwright-tests-chromium-sharded]
|
|
runs-on: ubuntu-latest
|
|
if: ${{ !cancelled() }}
|
|
steps:
|
|
- name: Checkout ComfyUI_frontend
|
|
uses: actions/checkout@v4
|
|
with:
|
|
repository: 'Comfy-Org/ComfyUI_frontend'
|
|
path: 'ComfyUI_frontend'
|
|
|
|
- name: Install pnpm
|
|
uses: pnpm/action-setup@v4
|
|
with:
|
|
version: 10
|
|
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: lts/*
|
|
cache: 'pnpm'
|
|
cache-dependency-path: 'ComfyUI_frontend/pnpm-lock.yaml'
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
pnpm install --frozen-lockfile
|
|
working-directory: ComfyUI_frontend
|
|
|
|
- name: Download blob reports
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
path: ComfyUI_frontend/all-blob-reports
|
|
pattern: blob-report-chromium-*
|
|
merge-multiple: true
|
|
|
|
- name: Merge into HTML Report
|
|
run: npx playwright merge-reports --reporter html ./all-blob-reports
|
|
working-directory: ComfyUI_frontend
|
|
|
|
- name: Upload HTML report
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: playwright-report-chromium
|
|
path: ComfyUI_frontend/playwright-report/
|
|
retention-days: 30
|
|
|
|
- name: Save deployment info for merged chromium report
|
|
if: always()
|
|
run: |
|
|
# Determine exit code based on the needs context
|
|
# The merge-reports job only runs if sharded tests complete (success, failure, or cancelled)
|
|
# We consider it successful only if all sharded tests succeeded
|
|
if [ "${{ needs.playwright-tests-chromium-sharded.result }}" = "success" ]; then
|
|
EXIT_CODE=0
|
|
else
|
|
EXIT_CODE=1
|
|
fi
|
|
|
|
echo "chromium|${EXIT_CODE}|https://comfyui-playwright-chromium.pages.dev" > deployment-info-chromium.txt
|
|
working-directory: ComfyUI_frontend
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: deployment-info-chromium
|
|
path: ComfyUI_frontend/deployment-info-chromium.txt
|
|
retention-days: 1
|