Compare commits

...

4 Commits

Author SHA1 Message Date
Alexander Brown
dd3f395c35 Merge branch 'main' into unified-pr-comments 2026-05-18 16:12:38 -07:00
Connor Byrne
6396bb9ee3 fix: ensure playwright section upsert runs after deploy failures
Add always() to Read and Upsert steps so the unified comment is updated
even when the deploy step fails, preventing stuck "Running..." state.

Addresses review feedback:
https://github.com/Comfy-Org/ComfyUI_frontend/pull/11935#discussion_r3221603536
2026-05-13 13:09:13 -07:00
bymyself
08f9c5b84f fix: move branch name expressions from run scripts into env vars
Prevents shell injection — ${{ github.head_ref }} and
${{ github.event.workflow_run.head_branch }} must not be interpolated
directly into shell run steps, especially in fork contexts where the
value is attacker-controlled.
2026-05-11 12:26:22 -07:00
bymyself
f33fc17bdf ci: consolidate PR comments into single unified report
Merge Playwright and Storybook bot comments into the existing unified
<!-- COMFYUI_FRONTEND_PR_REPORT --> comment, reducing PR noise from
4-5 separate bot comments down to 1 (plus external CodeRabbit/Codecov).

- Playwright/Storybook scripts now write to SUMMARY_FILE instead of
  posting standalone comments when that env var is set
- All comment-posting jobs now use upsert-comment-section, inserting
  named sections (playwright, storybook, ci-metrics, chromatic) into
  the shared comment without clobbering each other
- pr-report.yaml cleans up legacy PLAYWRIGHT_TEST_STATUS and
  STORYBOOK_BUILD_STATUS comments on first run
- perf-report.ts collapses detailed metric tables in <details> by
  default; only the headline FPS/TBT summary and a brief regression
  count remain visible inline

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-11 12:26:19 -07:00
9 changed files with 226 additions and 103 deletions

View File

@@ -1,21 +1,22 @@
name: Upsert Comment Section
description: >
Manage a consolidated PR comment with independently-updatable sections.
All website CI workflows share the marker <!-- WEBSITE_CI_REPORT -->.
Valid section names: "e2e", "preview", "screenshot-update".
Multiple CI workflows can share the same comment by using the same
comment-marker and different section-names. Each workflow upserts only
its own section, leaving other sections intact.
inputs:
pr-number:
description: PR number to comment on
required: true
section-name:
description: 'Section identifier: "e2e", "preview", or "screenshot-update"'
description: 'Section identifier (e.g. "playwright", "storybook", "e2e", "preview")'
required: true
section-content:
description: Markdown content for this section
required: true
comment-marker:
description: Top-level HTML comment marker (must be <!-- WEBSITE_CI_REPORT --> for all callers)
description: Top-level HTML comment marker shared by all sections in this comment
required: true
token:
description: GitHub token with pull-requests write permission

View File

@@ -38,16 +38,15 @@ jobs:
id: pr
uses: ./.github/actions/resolve-pr-from-workflow-run
- name: Handle Test Start
- name: Handle Test Start — upsert playwright starting section
if: steps.pr.outputs.skip != 'true' && github.event.action == 'requested'
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
chmod +x scripts/cicd/pr-playwright-deploy-and-comment.sh
./scripts/cicd/pr-playwright-deploy-and-comment.sh \
"${{ steps.pr.outputs.number }}" \
"${{ github.event.workflow_run.head_branch }}" \
"starting"
uses: ./.github/actions/upsert-comment-section
with:
pr-number: ${{ steps.pr.outputs.number }}
section-name: playwright
section-content: '## 🎭 Playwright: ⏳ Running...'
comment-marker: '<!-- COMFYUI_FRONTEND_PR_REPORT -->'
token: ${{ github.token }}
- name: Download and Deploy Reports
if: steps.pr.outputs.skip != 'true' && github.event.action == 'completed'
@@ -59,13 +58,14 @@ jobs:
path: reports
if_no_artifact_found: warn
- name: Handle Test Completion
- name: Handle Test Completion — deploy and generate section
if: steps.pr.outputs.skip != 'true' && github.event.action == 'completed' && hashFiles('reports/**') != ''
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
GITHUB_TOKEN: ${{ github.token }}
GITHUB_SHA: ${{ github.event.workflow_run.head_sha }}
SUMMARY_FILE: playwright-section.md
run: |
# Rename merged report if exists
[ -d "reports/playwright-report-chromium-merged" ] && \
@@ -76,3 +76,20 @@ jobs:
"${{ steps.pr.outputs.number }}" \
"${{ github.event.workflow_run.head_branch }}" \
"completed"
- name: Read playwright section
id: section
if: steps.pr.outputs.skip != 'true' && github.event.action == 'completed' && hashFiles('playwright-section.md') != ''
uses: juliangruber/read-file-action@b549046febe0fe86f8cb4f93c24e284433f9ab58 # v1.1.7
with:
path: playwright-section.md
- name: Upsert playwright section into unified report
if: steps.pr.outputs.skip != 'true' && github.event.action == 'completed' && steps.section.outputs.content != ''
uses: ./.github/actions/upsert-comment-section
with:
pr-number: ${{ steps.pr.outputs.number }}
section-name: playwright
section-content: ${{ steps.section.outputs.content }}
comment-marker: '<!-- COMFYUI_FRONTEND_PR_REPORT -->'
token: ${{ github.token }}

View File

@@ -228,7 +228,7 @@ jobs:
# when using pull_request event, we have permission to comment directly
# if its a forked repo, we need to use workflow_run event in a separate workflow (pr-playwright-deploy.yaml)
# Post starting comment for non-forked PRs
# Post starting section into the unified PR report comment for non-forked PRs
comment-on-pr-start:
needs: changes
runs-on: ubuntu-latest
@@ -244,17 +244,16 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v6
- name: Post starting comment
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
chmod +x scripts/cicd/pr-playwright-deploy-and-comment.sh
./scripts/cicd/pr-playwright-deploy-and-comment.sh \
"${{ github.event.pull_request.number }}" \
"${{ github.head_ref }}" \
"starting"
- name: Upsert playwright starting section into unified report
uses: ./.github/actions/upsert-comment-section
with:
pr-number: ${{ github.event.pull_request.number }}
section-name: playwright
section-content: '## 🎭 Playwright: ⏳ Running...'
comment-marker: '<!-- COMFYUI_FRONTEND_PR_REPORT -->'
token: ${{ github.token }}
# Deploy and comment for non-forked PRs only
# Deploy and upsert final playwright section for non-forked PRs only
deploy-and-comment:
needs: [changes, playwright-tests, merge-reports]
runs-on: ubuntu-latest
@@ -278,15 +277,34 @@ jobs:
pattern: playwright-report-*
path: reports
- name: Deploy reports and comment on PR
- name: Deploy reports and generate section
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
GITHUB_TOKEN: ${{ github.token }}
GITHUB_SHA: ${{ github.event.pull_request.head.sha }}
SUMMARY_FILE: playwright-section.md
BRANCH_NAME: ${{ github.head_ref }}
run: |
bash ./scripts/cicd/pr-playwright-deploy-and-comment.sh \
"${{ github.event.pull_request.number }}" \
"${{ github.head_ref }}" \
"$BRANCH_NAME" \
"completed"
- name: Read playwright section
id: section
if: always() && hashFiles('playwright-section.md') != ''
uses: juliangruber/read-file-action@b549046febe0fe86f8cb4f93c24e284433f9ab58 # v1.1.7
with:
path: playwright-section.md
- name: Upsert playwright section into unified report
if: always() && steps.section.outputs.content != ''
uses: ./.github/actions/upsert-comment-section
with:
pr-number: ${{ github.event.pull_request.number }}
section-name: playwright
section-content: ${{ steps.section.outputs.content }}
comment-marker: '<!-- COMFYUI_FRONTEND_PR_REPORT -->'
token: ${{ github.token }}
#### END Deployment and commenting (non-forked PRs only)

View File

@@ -38,16 +38,15 @@ jobs:
id: pr
uses: ./.github/actions/resolve-pr-from-workflow-run
- name: Handle Storybook Start
- name: Handle Storybook Start — upsert storybook starting section
if: steps.pr.outputs.skip != 'true' && github.event.action == 'requested'
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
chmod +x scripts/cicd/pr-storybook-deploy-and-comment.sh
./scripts/cicd/pr-storybook-deploy-and-comment.sh \
"${{ steps.pr.outputs.number }}" \
"${{ github.event.workflow_run.head_branch }}" \
"starting"
uses: ./.github/actions/upsert-comment-section
with:
pr-number: ${{ steps.pr.outputs.number }}
section-name: storybook
section-content: '## 🎨 Storybook: 🚧 Building...'
comment-marker: '<!-- COMFYUI_FRONTEND_PR_REPORT -->'
token: ${{ github.token }}
- name: Download and Deploy Storybook
if: steps.pr.outputs.skip != 'true' && github.event.action == 'completed' && github.event.workflow_run.conclusion == 'success'
@@ -58,7 +57,7 @@ jobs:
name: storybook-static
path: storybook-static
- name: Handle Storybook Completion
- name: Handle Storybook Completion — deploy and generate section
if: steps.pr.outputs.skip != 'true' && github.event.action == 'completed'
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
@@ -66,9 +65,28 @@ jobs:
GITHUB_TOKEN: ${{ github.token }}
WORKFLOW_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
WORKFLOW_URL: ${{ github.event.workflow_run.html_url }}
SUMMARY_FILE: storybook-section.md
BRANCH_NAME: ${{ github.event.workflow_run.head_branch }}
run: |
chmod +x scripts/cicd/pr-storybook-deploy-and-comment.sh
./scripts/cicd/pr-storybook-deploy-and-comment.sh \
"${{ steps.pr.outputs.number }}" \
"${{ github.event.workflow_run.head_branch }}" \
"$BRANCH_NAME" \
"completed"
- name: Read storybook section
id: section
if: steps.pr.outputs.skip != 'true' && github.event.action == 'completed' && hashFiles('storybook-section.md') != ''
uses: juliangruber/read-file-action@b549046febe0fe86f8cb4f93c24e284433f9ab58 # v1.1.7
with:
path: storybook-section.md
- name: Upsert storybook section into unified report
if: steps.pr.outputs.skip != 'true' && github.event.action == 'completed' && steps.section.outputs.content != ''
uses: ./.github/actions/upsert-comment-section
with:
pr-number: ${{ steps.pr.outputs.number }}
section-name: storybook
section-content: ${{ steps.section.outputs.content }}
comment-marker: '<!-- COMFYUI_FRONTEND_PR_REPORT -->'
token: ${{ github.token }}

View File

@@ -37,15 +37,14 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v6
- name: Post starting comment
env:
GITHUB_TOKEN: ${{ github.token }}
run: |
chmod +x scripts/cicd/pr-storybook-deploy-and-comment.sh
./scripts/cicd/pr-storybook-deploy-and-comment.sh \
"${{ github.event.pull_request.number }}" \
"${{ github.head_ref }}" \
"starting"
- name: Upsert storybook starting section into unified report
uses: ./.github/actions/upsert-comment-section
with:
pr-number: ${{ github.event.pull_request.number }}
section-name: storybook
section-content: '## 🎨 Storybook: 🚧 Building...'
comment-marker: '<!-- COMFYUI_FRONTEND_PR_REPORT -->'
token: ${{ github.token }}
# Build Storybook for all PRs (free Cloudflare deployment)
storybook-build:
@@ -164,19 +163,38 @@ jobs:
- name: Make deployment script executable
run: chmod +x scripts/cicd/pr-storybook-deploy-and-comment.sh
- name: Deploy Storybook and comment on PR
- name: Deploy Storybook and generate section
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
GITHUB_TOKEN: ${{ github.token }}
WORKFLOW_CONCLUSION: ${{ needs.storybook-build.outputs.conclusion }}
WORKFLOW_URL: ${{ needs.storybook-build.outputs.workflow-url }}
SUMMARY_FILE: storybook-section.md
BRANCH_NAME: ${{ github.head_ref }}
run: |
./scripts/cicd/pr-storybook-deploy-and-comment.sh \
"${{ github.event.pull_request.number }}" \
"${{ github.head_ref }}" \
"$BRANCH_NAME" \
"completed"
- name: Read storybook section
id: section
if: hashFiles('storybook-section.md') != ''
uses: juliangruber/read-file-action@b549046febe0fe86f8cb4f93c24e284433f9ab58 # v1.1.7
with:
path: storybook-section.md
- name: Upsert storybook section into unified report
if: steps.section.outputs.content != ''
uses: ./.github/actions/upsert-comment-section
with:
pr-number: ${{ github.event.pull_request.number }}
section-name: storybook
section-content: ${{ steps.section.outputs.content }}
comment-marker: '<!-- COMFYUI_FRONTEND_PR_REPORT -->'
token: ${{ github.token }}
# Deploy Storybook to production URL on main branch push
deploy-production:
runs-on: ubuntu-latest
@@ -208,35 +226,14 @@ jobs:
permissions:
pull-requests: write
steps:
- name: Update comment with Chromatic URLs
uses: actions/github-script@v8
- name: Upsert Chromatic section into unified report
uses: ./.github/actions/upsert-comment-section
with:
script: |
const buildUrl = '${{ needs.chromatic-deployment.outputs.chromatic-build-url }}';
const storybookUrl = '${{ needs.chromatic-deployment.outputs.chromatic-storybook-url }}';
// Find the existing Storybook comment
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: ${{ github.event.pull_request.number }}
});
const storybookComment = comments.find(comment =>
comment.body.includes('<!-- STORYBOOK_BUILD_STATUS -->')
);
if (storybookComment && buildUrl && storybookUrl) {
// Append Chromatic info to existing comment
const updatedBody = storybookComment.body.replace(
/---\n(.*)$/s,
`---\n### 🎨 Chromatic Visual Tests\n- 📊 [View Chromatic Build](${buildUrl})\n- 📚 [View Chromatic Storybook](${storybookUrl})\n\n$1`
);
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: storybookComment.id,
body: updatedBody
});
}
pr-number: ${{ github.event.pull_request.number }}
section-name: chromatic
section-content: |
### 🎨 Chromatic Visual Tests
- 📊 [View Chromatic Build](${{ needs.chromatic-deployment.outputs.chromatic-build-url }})
- 📚 [View Chromatic Storybook](${{ needs.chromatic-deployment.outputs.chromatic-storybook-url }})
comment-marker: '<!-- COMFYUI_FRONTEND_PR_REPORT -->'
token: ${{ github.token }}

View File

@@ -138,6 +138,8 @@ jobs:
const legacyMarkers = [
'<!-- COMFYUI_FRONTEND_SIZE -->',
'<!-- COMFYUI_FRONTEND_PERF -->',
'<!-- PLAYWRIGHT_TEST_STATUS -->',
'<!-- STORYBOOK_BUILD_STATUS -->',
];
const comments = await github.paginate(github.rest.issues.listComments, {
@@ -158,11 +160,19 @@ jobs:
}
}
- name: Post PR comment
- name: Read PR report
id: report
if: steps.pr-meta.outputs.skip != 'true'
uses: ./.github/actions/post-pr-report-comment
uses: juliangruber/read-file-action@b549046febe0fe86f8cb4f93c24e284433f9ab58 # v1.1.7
with:
path: ./pr-report.md
- name: Upsert bundle/perf/coverage section into unified report
if: steps.pr-meta.outputs.skip != 'true'
uses: ./.github/actions/upsert-comment-section
with:
pr-number: ${{ steps.pr-meta.outputs.number }}
report-file: ./pr-report.md
section-name: ci-metrics
section-content: ${{ steps.report.outputs.content }}
comment-marker: '<!-- COMFYUI_FRONTEND_PR_REPORT -->'
token: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -1,8 +1,13 @@
#!/bin/bash
set -e
# Deploy Playwright test reports to Cloudflare Pages and comment on PR
# Deploy Playwright test reports to Cloudflare Pages and write section markdown.
# Usage: ./pr-playwright-deploy-and-comment.sh <pr_number> <branch_name> <status>
#
# When SUMMARY_FILE env var is set, the generated markdown is written there
# instead of posted as a standalone GitHub comment. The caller is then
# responsible for upserting that content into the unified PR report via the
# upsert-comment-section action.
# Input validation
# Validate PR number is numeric
@@ -103,17 +108,26 @@ deploy_report() {
echo "failed"
}
# Post or update GitHub comment
# Post or update GitHub comment, or write to SUMMARY_FILE if set.
# When SUMMARY_FILE is set, the caller (workflow) is responsible for upserting
# the content into the unified PR report via upsert-comment-section.
post_comment() {
body="$1"
if [ -n "${SUMMARY_FILE:-}" ]; then
printf '%s\n' "$body" > "$SUMMARY_FILE"
echo "Wrote playwright section to $SUMMARY_FILE" >&2
return
fi
temp_file=$(mktemp)
echo "$body" > "$temp_file"
if command -v gh > /dev/null 2>&1; then
# Find existing comment ID
existing=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \
--jq ".[] | select(.body | contains(\"$COMMENT_MARKER\")) | .id" | head -1)
if [ -n "$existing" ]; then
# Update specific comment by ID
gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" \
@@ -126,15 +140,20 @@ post_comment() {
echo "GitHub CLI not available, outputting comment:"
cat "$temp_file"
fi
rm -f "$temp_file"
}
# Main execution
if [ "$STATUS" = "starting" ]; then
# Post concise starting comment
comment="$COMMENT_MARKER
# When writing to SUMMARY_FILE, omit the standalone marker (the upsert
# action uses its own section delimiters).
if [ -n "${SUMMARY_FILE:-}" ]; then
comment="## 🎭 Playwright: ⏳ Running..."
else
comment="$COMMENT_MARKER
## 🎭 Playwright: ⏳ Running..."
fi
post_comment "$comment"
else
@@ -281,9 +300,14 @@ else
flaky_note=" · $total_flaky flaky"
fi
# Generate compact single-line comment
comment="$COMMENT_MARKER
# Generate compact single-line comment (omit standalone marker when writing
# to SUMMARY_FILE — the upsert action adds its own section delimiters).
if [ -n "${SUMMARY_FILE:-}" ]; then
comment="## 🎭 Playwright: $status_icon $total_passed passed, $total_failed failed$flaky_note"
else
comment="$COMMENT_MARKER
## 🎭 Playwright: $status_icon $total_passed passed, $total_failed failed$flaky_note"
fi
# Extract and display failed tests from all browsers (flaky tests are treated as passing)
if [ $total_failed -gt 0 ]; then

View File

@@ -1,8 +1,13 @@
#!/bin/bash
set -e
# Deploy Storybook to Cloudflare Pages and comment on PR
# Deploy Storybook to Cloudflare Pages and write section markdown.
# Usage: ./pr-storybook-deploy-and-comment.sh <pr_number> <branch_name> <status>
#
# When SUMMARY_FILE env var is set, the generated markdown is written there
# instead of posted as a standalone GitHub comment. The caller is then
# responsible for upserting that content into the unified PR report via the
# upsert-comment-section action.
# Input validation
# Validate PR number is numeric
@@ -91,17 +96,26 @@ deploy_storybook() {
echo "failed"
}
# Post or update GitHub comment
# Post or update GitHub comment, or write to SUMMARY_FILE if set.
# When SUMMARY_FILE is set, the caller (workflow) is responsible for upserting
# the content into the unified PR report via upsert-comment-section.
post_comment() {
body="$1"
if [ -n "${SUMMARY_FILE:-}" ]; then
printf '%s\n' "$body" > "$SUMMARY_FILE"
echo "Wrote storybook section to $SUMMARY_FILE" >&2
return
fi
temp_file=$(mktemp)
echo "$body" > "$temp_file"
if command -v gh > /dev/null 2>&1; then
# Find existing comment ID
existing=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \
--jq ".[] | select(.body | contains(\"$COMMENT_MARKER\")) | .id" | head -1)
if [ -n "$existing" ]; then
# Update specific comment by ID
gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" \
@@ -113,15 +127,20 @@ post_comment() {
echo "GitHub CLI not available, outputting comment:"
cat "$temp_file"
fi
rm -f "$temp_file"
}
# Main execution
if [ "$STATUS" = "starting" ]; then
# Post starting comment
comment="$COMMENT_MARKER
# When writing to SUMMARY_FILE, omit the standalone marker (the upsert
# action uses its own section delimiters).
if [ -n "${SUMMARY_FILE:-}" ]; then
comment="## 🎨 Storybook: <img alt='loading' src='https://github.com/user-attachments/assets/755c86ee-e445-4ea8-bc2c-cca85df48686' width='14px' height='14px'/> Building..."
else
comment="$COMMENT_MARKER
## 🎨 Storybook: <img alt='loading' src='https://github.com/user-attachments/assets/755c86ee-e445-4ea8-bc2c-cca85df48686' width='14px' height='14px'/> Building..."
fi
post_comment "$comment"
elif [ "$STATUS" = "completed" ]; then
@@ -197,10 +216,18 @@ elif [ "$STATUS" = "completed" ]; then
</details>"
comment="$COMMENT_MARKER
# Omit standalone marker when writing to SUMMARY_FILE — the upsert action
# adds its own section delimiters.
if [ -n "${SUMMARY_FILE:-}" ]; then
comment="$header
$details"
else
comment="$COMMENT_MARKER
$header
$details"
fi
post_comment "$comment"
fi

View File

@@ -316,12 +316,16 @@ function renderFullReport(
lines.push(
`⚠️ **${flaggedRows.length} regression${flaggedRows.length > 1 ? 's' : ''} detected**`,
'',
`<details><summary>Show regressions</summary>`,
'',
...tableHeader,
...flaggedRows,
'',
'</details>',
''
)
} else {
lines.push('No regressions detected.', '')
lines.push('No regressions detected.', '')
}
lines.push(
@@ -393,6 +397,8 @@ function renderColdStartReport(
lines.push(
`> Collecting baseline variance data (${historicalCount}/15 runs). Significance will appear after 2 main branch runs.`,
'',
`<details><summary>All metrics (cold start)</summary>`,
'',
'| Metric | Baseline | PR | Δ |',
'|--------|----------|-----|---|'
)
@@ -430,6 +436,7 @@ function renderColdStartReport(
}
}
lines.push('', '</details>')
return lines
}
@@ -438,7 +445,10 @@ function renderNoBaselineReport(
): string[] {
const lines: string[] = []
lines.push(
'No baseline found — showing absolute values.\n',
'> No baseline found — significance unavailable.',
'',
`<details><summary>Absolute values</summary>`,
'',
'| Metric | Value |',
'|--------|-------|'
)
@@ -449,6 +459,7 @@ function renderNoBaselineReport(
lines.push(`| ${testName}: ${label} | ${formatValue(prVal, unit)} |`)
}
}
lines.push('', '</details>')
return lines
}