mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-02-22 07:44:11 +00:00
df6723415b
- Add workflow documentation explaining selective update strategy - Improve logging with clear output formatting (no emojis) - Add GitHub Actions workflow summary with file change details - Fix command injection vulnerability by validating test paths with regex - Add error handling for JSON.parse with descriptive messages - Replace non-null assertion with safer null checking pattern - Add explicit error handling for TypeScript script execution
251 lines
10 KiB
YAML
251 lines
10 KiB
YAML
# Setting test expectation screenshots for Playwright
|
|
#
|
|
# This workflow uses a selective snapshot update strategy:
|
|
# 1. When tests fail in CI, they generate a manifest of failed test locations (file:line)
|
|
# 2. This workflow downloads that manifest from the failed test run artifacts
|
|
# 3. Only the failed tests are re-run with --update-snapshots (much faster than running all tests)
|
|
# 4. Updated snapshots are committed back to the PR branch
|
|
#
|
|
# Trigger: Add label "New Browser Test Expectations" OR comment "/update-playwright" on PR
|
|
name: Update Playwright Expectations
|
|
|
|
on:
|
|
pull_request:
|
|
types: [labeled]
|
|
issue_comment:
|
|
types: [created]
|
|
|
|
jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
if: >
|
|
( github.event_name == 'pull_request' && github.event.label.name == 'New Browser Test Expectations' ) ||
|
|
( github.event.issue.pull_request &&
|
|
github.event_name == 'issue_comment' &&
|
|
(
|
|
github.event.comment.author_association == 'OWNER' ||
|
|
github.event.comment.author_association == 'MEMBER' ||
|
|
github.event.comment.author_association == 'COLLABORATOR'
|
|
) &&
|
|
startsWith(github.event.comment.body, '/update-playwright') )
|
|
steps:
|
|
- name: Initial Checkout
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Pull Request Checkout
|
|
if: github.event.issue.pull_request && github.event_name == 'issue_comment'
|
|
run: gh pr checkout ${{ github.event.issue.number }}
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Setup Frontend
|
|
uses: ./.github/actions/setup-frontend
|
|
|
|
- name: Setup Playwright
|
|
uses: ./.github/actions/setup-playwright
|
|
|
|
- name: Locate failed screenshot manifest artifact
|
|
id: locate-manifest
|
|
uses: actions/github-script@v8
|
|
with:
|
|
script: |
|
|
const { owner, repo } = context.repo
|
|
let headSha = ''
|
|
if (context.eventName === 'pull_request') {
|
|
headSha = context.payload.pull_request.head.sha
|
|
} else if (context.eventName === 'issue_comment') {
|
|
const prNumber = context.payload.issue.number
|
|
const pr = await github.rest.pulls.get({ owner, repo, pull_number: prNumber })
|
|
headSha = pr.data.head.sha
|
|
}
|
|
|
|
if (!headSha) {
|
|
core.setOutput('run_id', '')
|
|
core.setOutput('has_manifest', 'false')
|
|
return
|
|
}
|
|
|
|
const { data } = await github.rest.actions.listWorkflowRuns({
|
|
owner,
|
|
repo,
|
|
workflow_id: 'tests-ci.yaml',
|
|
head_sha: headSha,
|
|
event: 'pull_request',
|
|
per_page: 1,
|
|
})
|
|
const run = data.workflow_runs?.[0]
|
|
|
|
let has = 'false'
|
|
let runId = ''
|
|
if (run) {
|
|
runId = String(run.id)
|
|
const { data: { artifacts = [] } } = await github.rest.actions.listWorkflowRunArtifacts({
|
|
owner,
|
|
repo,
|
|
run_id: run.id,
|
|
per_page: 100,
|
|
})
|
|
if (artifacts.some(a => a.name === 'failed-screenshot-tests' && !a.expired)) has = 'true'
|
|
}
|
|
core.setOutput('run_id', runId)
|
|
core.setOutput('has_manifest', has)
|
|
|
|
- name: Download failed screenshot manifest
|
|
if: steps.locate-manifest.outputs.has_manifest == 'true'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
run-id: ${{ steps.locate-manifest.outputs.run_id }}
|
|
name: failed-screenshot-tests
|
|
path: ComfyUI_frontend/ci-rerun
|
|
|
|
- name: Re-run failed screenshot tests and update snapshots
|
|
id: playwright-tests
|
|
shell: bash
|
|
working-directory: ComfyUI_frontend
|
|
continue-on-error: true
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "Selective Snapshot Update"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
# Check if manifest exists
|
|
if [ ! -d ci-rerun ]; then
|
|
echo "ERROR: No manifest found in ci-rerun/ directory"
|
|
echo " This means no failed screenshot tests were detected in the latest CI run."
|
|
echo " Please ensure tests have been run and failures were recorded."
|
|
exit 1
|
|
fi
|
|
|
|
shopt -s nullglob
|
|
files=(ci-rerun/*.txt)
|
|
|
|
if [ ${#files[@]} -eq 0 ]; then
|
|
echo "ERROR: No manifest files found in ci-rerun/"
|
|
echo " Expected files like: chromium.txt, chromium-2x.txt, mobile-chrome.txt"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Found ${#files[@]} project manifest(s):"
|
|
for f in "${files[@]}"; do
|
|
project="$(basename "$f" .txt)"
|
|
count=$(grep -c . "$f" 2>/dev/null || echo "0")
|
|
echo " - $project: $count failed test(s)"
|
|
done
|
|
echo ""
|
|
|
|
# Re-run tests per project
|
|
total_tests=0
|
|
for f in "${files[@]}"; do
|
|
project="$(basename "$f" .txt)"
|
|
mapfile -t lines < "$f"
|
|
filtered=( )
|
|
|
|
# Validate and sanitize test paths to prevent command injection
|
|
for l in "${lines[@]}"; do
|
|
# Skip empty lines
|
|
[ -z "$l" ] && continue
|
|
|
|
# Validate format: must be browser_tests/...spec.ts:number
|
|
if [[ "$l" =~ ^browser_tests/.+\.spec\.ts:[0-9]+$ ]]; then
|
|
filtered+=("$l")
|
|
else
|
|
echo "WARNING: Skipping invalid test path: $l"
|
|
fi
|
|
done
|
|
|
|
if [ ${#filtered[@]} -eq 0 ]; then
|
|
echo "WARNING: Skipping $project (no valid tests in manifest)"
|
|
continue
|
|
fi
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "Updating snapshots for project: $project"
|
|
echo " Re-running ${#filtered[@]} failed test(s)..."
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
|
|
pnpm exec playwright test --project="$project" --update-snapshots \
|
|
--reporter=line --reporter=html \
|
|
"${filtered[@]}"
|
|
|
|
total_tests=$((total_tests + ${#filtered[@]}))
|
|
echo ""
|
|
done
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "Completed snapshot updates for $total_tests test(s)"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
- uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: playwright-report
|
|
path: ComfyUI_frontend/playwright-report/
|
|
retention-days: 30
|
|
|
|
- name: Debugging info
|
|
working-directory: ComfyUI_frontend
|
|
run: |
|
|
echo "Branch: ${{ github.head_ref }}"
|
|
git status
|
|
|
|
- name: Commit updated expectations
|
|
id: commit
|
|
working-directory: ComfyUI_frontend
|
|
run: |
|
|
git config --global user.name 'github-actions'
|
|
git config --global user.email 'github-actions@github.com'
|
|
if [ "${{ github.event_name }}" = "issue_comment" ]; then
|
|
true
|
|
else
|
|
git fetch origin ${{ github.head_ref }}
|
|
git checkout -B ${{ github.head_ref }} origin/${{ github.head_ref }}
|
|
fi
|
|
git add browser_tests
|
|
if git diff --cached --quiet; then
|
|
echo "No expectation updates detected; skipping commit."
|
|
echo "changed=false" >> $GITHUB_OUTPUT
|
|
else
|
|
# Count changed snapshots
|
|
changed_count=$(git diff --cached --name-only browser_tests | wc -l)
|
|
echo "changed=true" >> $GITHUB_OUTPUT
|
|
echo "count=$changed_count" >> $GITHUB_OUTPUT
|
|
|
|
git commit -m "[automated] Update test expectations"
|
|
if [ "${{ github.event_name }}" = "issue_comment" ]; then
|
|
git push
|
|
else
|
|
git push origin HEAD:${{ github.head_ref }}
|
|
fi
|
|
fi
|
|
|
|
- name: Generate workflow summary
|
|
if: always()
|
|
working-directory: ComfyUI_frontend
|
|
run: |
|
|
echo "## Snapshot Update Summary" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
|
|
if [ "${{ steps.commit.outputs.changed }}" = "true" ]; then
|
|
echo "**${{ steps.commit.outputs.count }} snapshot(s) updated**" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "<details>" >> $GITHUB_STEP_SUMMARY
|
|
echo "<summary>View updated files</summary>" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
git diff HEAD~1 --name-only browser_tests 2>/dev/null || echo "No git history available" >> $GITHUB_STEP_SUMMARY
|
|
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
|
echo "</details>" >> $GITHUB_STEP_SUMMARY
|
|
elif [ "${{ steps.commit.outputs.changed }}" = "false" ]; then
|
|
echo "No snapshot changes detected" >> $GITHUB_STEP_SUMMARY
|
|
else
|
|
echo "WARNING: Snapshot update may have failed - check logs above" >> $GITHUB_STEP_SUMMARY
|
|
fi
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "---" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "**Strategy:** Selective snapshot update (only failed tests re-run)" >> $GITHUB_STEP_SUMMARY
|