mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-02-28 18:54:09 +00:00
939d1a0e44
Resolved conflicts in update-playwright-expectations.yaml by: - Keeping main's improvements: concurrency control, comment reactions, better branch checkout - Keeping our selective snapshot update logic with validation - Keeping our workflow summary generation - Combined both sets of improvements for a robust solution Fixed eslint configuration issue where vite.config.mts was in both allowDefaultProject and tsconfig.json
285 lines
12 KiB
YAML
285 lines
12 KiB
YAML
# Setting test expectation screenshots for Playwright
|
|
#
|
|
# This workflow uses a selective snapshot update strategy:
|
|
# 1. When tests fail in CI, they generate a manifest of failed test locations (file:line)
|
|
# 2. This workflow downloads that manifest from the failed test run artifacts
|
|
# 3. Only the failed tests are re-run with --update-snapshots (much faster than running all tests)
|
|
# 4. Updated snapshots are committed back to the PR branch
|
|
#
|
|
# Trigger: Add label "New Browser Test Expectations" OR comment "/update-playwright" on PR
|
|
name: Update Playwright Expectations
|
|
|
|
on:
|
|
pull_request:
|
|
types: [labeled]
|
|
issue_comment:
|
|
types: [created]
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
if: >
|
|
( github.event_name == 'pull_request' && github.event.label.name == 'New Browser Test Expectations' ) ||
|
|
( github.event.issue.pull_request &&
|
|
github.event_name == 'issue_comment' &&
|
|
(
|
|
github.event.comment.author_association == 'OWNER' ||
|
|
github.event.comment.author_association == 'MEMBER' ||
|
|
github.event.comment.author_association == 'COLLABORATOR'
|
|
) &&
|
|
startsWith(github.event.comment.body, '/update-playwright') )
|
|
steps:
|
|
- name: Find Update Comment
|
|
uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad
|
|
id: "find-update-comment"
|
|
with:
|
|
issue-number: ${{ github.event.number || github.event.issue.number }}
|
|
comment-author: "github-actions[bot]"
|
|
body-includes: "Updating Playwright Expectations"
|
|
|
|
- name: Add Starting Reaction
|
|
uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9
|
|
with:
|
|
comment-id: ${{ steps.find-update-comment.outputs.comment-id }}
|
|
issue-number: ${{ github.event.number || github.event.issue.number }}
|
|
body: |
|
|
Updating Playwright Expectations
|
|
edit-mode: replace
|
|
reactions: eyes
|
|
|
|
- name: Get Branch SHA
|
|
id: "get-branch"
|
|
run: echo ::set-output name=branch::$(gh pr view $PR_NO --repo $REPO --json headRefName --jq '.headRefName')
|
|
env:
|
|
REPO: ${{ github.repository }}
|
|
PR_NO: ${{ github.event.number || github.event.issue.number }}
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Initial Checkout
|
|
uses: actions/checkout@v5
|
|
with:
|
|
ref: ${{ steps.get-branch.outputs.branch }}
|
|
|
|
- name: Setup Frontend
|
|
uses: ./.github/actions/setup-frontend
|
|
with:
|
|
include_build_step: true
|
|
|
|
- name: Setup ComfyUI Server
|
|
uses: ./.github/actions/setup-comfyui-server
|
|
with:
|
|
launch_server: true
|
|
|
|
- name: Setup Playwright
|
|
uses: ./.github/actions/setup-playwright
|
|
|
|
- name: Locate failed screenshot manifest artifact
|
|
id: locate-manifest
|
|
uses: actions/github-script@v8
|
|
with:
|
|
script: |
|
|
const { owner, repo } = context.repo
|
|
let headSha = ''
|
|
if (context.eventName === 'pull_request') {
|
|
headSha = context.payload.pull_request.head.sha
|
|
} else if (context.eventName === 'issue_comment') {
|
|
const prNumber = context.payload.issue.number
|
|
const pr = await github.rest.pulls.get({ owner, repo, pull_number: prNumber })
|
|
headSha = pr.data.head.sha
|
|
}
|
|
|
|
if (!headSha) {
|
|
core.setOutput('run_id', '')
|
|
core.setOutput('has_manifest', 'false')
|
|
return
|
|
}
|
|
|
|
const { data } = await github.rest.actions.listWorkflowRuns({
|
|
owner,
|
|
repo,
|
|
workflow_id: 'tests-ci.yaml',
|
|
head_sha: headSha,
|
|
event: 'pull_request',
|
|
per_page: 1,
|
|
})
|
|
const run = data.workflow_runs?.[0]
|
|
|
|
let has = 'false'
|
|
let runId = ''
|
|
if (run) {
|
|
runId = String(run.id)
|
|
const { data: { artifacts = [] } } = await github.rest.actions.listWorkflowRunArtifacts({
|
|
owner,
|
|
repo,
|
|
run_id: run.id,
|
|
per_page: 100,
|
|
})
|
|
if (artifacts.some(a => a.name === 'failed-screenshot-tests' && !a.expired)) has = 'true'
|
|
}
|
|
core.setOutput('run_id', runId)
|
|
core.setOutput('has_manifest', has)
|
|
|
|
- name: Download failed screenshot manifest
|
|
if: steps.locate-manifest.outputs.has_manifest == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
run-id: ${{ steps.locate-manifest.outputs.run_id }}
|
|
name: failed-screenshot-tests
|
|
path: ci-rerun
|
|
|
|
- name: Re-run failed screenshot tests and update snapshots
|
|
id: playwright-tests
|
|
continue-on-error: true
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "Selective Snapshot Update"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
# Check if manifest exists
|
|
if [ ! -d ci-rerun ]; then
|
|
echo "ERROR: No manifest found in ci-rerun/ directory"
|
|
echo " This means no failed screenshot tests were detected in the latest CI run."
|
|
echo " Please ensure tests have been run and failures were recorded."
|
|
exit 1
|
|
fi
|
|
|
|
shopt -s nullglob
|
|
files=(ci-rerun/*.txt)
|
|
|
|
if [ ${#files[@]} -eq 0 ]; then
|
|
echo "ERROR: No manifest files found in ci-rerun/"
|
|
echo " Expected files like: chromium.txt, chromium-2x.txt, mobile-chrome.txt"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Found ${#files[@]} project manifest(s):"
|
|
for f in "${files[@]}"; do
|
|
project="$(basename "$f" .txt)"
|
|
count=$(grep -c . "$f" 2>/dev/null || echo "0")
|
|
echo " - $project: $count failed test(s)"
|
|
done
|
|
echo ""
|
|
|
|
# Re-run tests per project
|
|
total_tests=0
|
|
for f in "${files[@]}"; do
|
|
project="$(basename "$f" .txt)"
|
|
mapfile -t lines < "$f"
|
|
filtered=( )
|
|
|
|
# Validate and sanitize test paths to prevent command injection
|
|
for l in "${lines[@]}"; do
|
|
# Skip empty lines
|
|
[ -z "$l" ] && continue
|
|
|
|
# Validate format: must be browser_tests/...spec.ts:number
|
|
if [[ "$l" =~ ^browser_tests/.+\.spec\.ts:[0-9]+$ ]]; then
|
|
filtered+=("$l")
|
|
else
|
|
echo "WARNING: Skipping invalid test path: $l"
|
|
fi
|
|
done
|
|
|
|
if [ ${#filtered[@]} -eq 0 ]; then
|
|
echo "WARNING: Skipping $project (no valid tests in manifest)"
|
|
continue
|
|
fi
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "Updating snapshots for project: $project"
|
|
echo " Re-running ${#filtered[@]} failed test(s)..."
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
|
|
pnpm exec playwright test --project="$project" --update-snapshots \
|
|
--reporter=line --reporter=html \
|
|
"${filtered[@]}"
|
|
|
|
total_tests=$((total_tests + ${#filtered[@]}))
|
|
echo ""
|
|
done
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "Completed snapshot updates for $total_tests test(s)"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: playwright-report
|
|
path: ./playwright-report/
|
|
retention-days: 30
|
|
|
|
- name: Debugging info
|
|
run: |
|
|
echo "PR: ${{ github.event.issue.number }}"
|
|
echo "Branch: ${{ steps.get-branch.outputs.branch }}"
|
|
git status
|
|
|
|
- name: Commit updated expectations
|
|
id: commit
|
|
run: |
|
|
git config --global user.name 'github-actions'
|
|
git config --global user.email 'github-actions@github.com'
|
|
git add browser_tests
|
|
if git diff --cached --quiet; then
|
|
echo "No expectation updates detected; skipping commit."
|
|
echo "changed=false" >> $GITHUB_OUTPUT
|
|
else
|
|
# Count changed snapshots
|
|
changed_count=$(git diff --cached --name-only browser_tests | wc -l)
|
|
echo "changed=true" >> $GITHUB_OUTPUT
|
|
echo "count=$changed_count" >> $GITHUB_OUTPUT
|
|
|
|
git commit -m "[automated] Update test expectations"
|
|
git push origin ${{ steps.get-branch.outputs.branch }}
|
|
fi
|
|
|
|
- name: Generate workflow summary
|
|
if: always()
|
|
run: |
|
|
echo "## Snapshot Update Summary" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
|
|
if [ "${{ steps.commit.outputs.changed }}" = "true" ]; then
|
|
echo "**${{ steps.commit.outputs.count }} snapshot(s) updated**" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "<details>" >> $GITHUB_STEP_SUMMARY
|
|
echo "<summary>View updated files</summary>" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
git diff HEAD~1 --name-only browser_tests 2>/dev/null || echo "No git history available" >> $GITHUB_STEP_SUMMARY
|
|
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
echo "</details>" >> $GITHUB_STEP_SUMMARY
|
|
elif [ "${{ steps.commit.outputs.changed }}" = "false" ]; then
|
|
echo "No snapshot changes detected" >> $GITHUB_STEP_SUMMARY
|
|
else
|
|
echo "WARNING: Snapshot update may have failed - check logs above" >> $GITHUB_STEP_SUMMARY
|
|
fi
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "---" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "**Strategy:** Selective snapshot update (only failed tests re-run)" >> $GITHUB_STEP_SUMMARY
|
|
|
|
- name: Add Done Reaction
|
|
uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9
|
|
if: github.event_name == 'issue_comment'
|
|
with:
|
|
comment-id: ${{ steps.find-update-comment.outputs.comment-id }}
|
|
issue-number: ${{ github.event.number || github.event.issue.number }}
|
|
reactions: +1
|
|
reactions-edit-mode: replace
|
|
|
|
- name: Remove New Browser Test Expectations label
|
|
if: always() && github.event_name == 'pull_request'
|
|
run: gh pr edit ${{ github.event.pull_request.number }} --remove-label "New Browser Test Expectations"
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|