Comfy-Org/ComfyUI_frontend
1
0
Fork 0
mirror of https://github.com/Comfy-Org/ComfyUI_frontend.git synced 2026-04-20 06:20:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: address Copilot review feedback on QA scripts

Browse Source 
- Enforce requestTimeoutMs via Gemini SDK requestOptions
- Add 100MB video size check before base64 encoding
- Sanitize screenshot filenames to prevent path traversal
- Sort video files by mtime for reliable rename
- Validate --mode arg against allowed values
- Add Content-Length pre-check in downloadMedia
- Add GitHub domain allowlist for media downloads (SSRF mitigation)
- Add contents:write permission and git config for report job
- Update Node.js requirement in SKILL.md from 18+ to 22+
This commit is contained in:
snomiao
2026-03-23 16:06:24 +00:00
parent f78f44fea1
commit 836492e3dc
6 changed files with 710 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.claude/skills/comfy-qa/SKILL.md
View File

@@ -9,7 +9,7 @@ Perform comprehensive quality assurance of the ComfyUI frontend application by n
## Prerequisites
- Node.js 18+
- Node.js 22+
- `pnpm` package manager
- `gh` CLI (authenticated)
- `playwright-cli` (browser automation): `npm install -g @playwright/cli@latest`

237
.github/workflows/pr-qa.yaml vendored
View File

@@ -36,6 +36,10 @@ jobs:
os: ${{ steps.set.outputs.os }}
mode: ${{ steps.set.outputs.mode }}
skip: ${{ steps.set.outputs.skip }}
number: ${{ steps.resolve-number.outputs.number }}
target_type: ${{ steps.resolve-number.outputs.target_type }}
before_sha: ${{ steps.resolve-refs.outputs.before_sha }}
after_sha: ${{ steps.resolve-refs.outputs.after_sha }}
steps:
- name: Determine QA mode
id: set
@@ -71,6 +75,54 @@ jobs:
fi
echo "Mode: $([ "$FULL" = "true" ] && echo full || echo focused)"
- name: Resolve target number and type
id: resolve-number
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUM: ${{ github.event.pull_request.number }}
BRANCH: ${{ github.ref_name }}
REPO: ${{ github.repository }}
run: |
if [ -n "$PR_NUM" ]; then
NUM="$PR_NUM"
else
NUM=$(gh pr list --repo "$REPO" \
--head "$BRANCH" --state open \
--json number --jq '.[0].number // empty')
if [ -z "$NUM" ]; then
NUM=$(echo "$BRANCH" | sed -n 's/^sno-qa-\([0-9]\+\)$/\1/p')
fi
fi
echo "number=${NUM}" >> "$GITHUB_OUTPUT"
if [ -n "$NUM" ]; then
if gh pr view "$NUM" --repo "$REPO" --json number >/dev/null 2>&1; then
echo "target_type=pr" >> "$GITHUB_OUTPUT"
echo "Target: PR #$NUM"
else
echo "target_type=issue" >> "$GITHUB_OUTPUT"
echo "Target: Issue #$NUM"
fi
fi
- name: Resolve commit SHAs for immutable references
id: resolve-refs
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NUM: ${{ steps.resolve-number.outputs.number }}
TARGET_TYPE: ${{ steps.resolve-number.outputs.target_type }}
REPO: ${{ github.repository }}
run: |
MAIN_SHA=$(gh api "repos/${REPO}/git/ref/heads/main" --jq '.object.sha')
echo "before_sha=${MAIN_SHA}" >> "$GITHUB_OUTPUT"
echo "Main: ${MAIN_SHA:0:7}"
if [ "$TARGET_TYPE" = "pr" ] && [ -n "$NUM" ]; then
PR_SHA=$(gh pr view "$NUM" --repo "$REPO" --json headRefOid --jq '.headRefOid')
echo "after_sha=${PR_SHA}" >> "$GITHUB_OUTPUT"
echo "PR #${NUM}: ${PR_SHA:0:7}"
fi
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# Analyze PR — deep analysis via Gemini Pro to generate QA guides
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
@@ -90,41 +142,21 @@ jobs:
with:
include_build_step: false
- name: Resolve PR number
id: pr
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUM: ${{ github.event.pull_request.number }}
BRANCH: ${{ github.ref_name }}
run: |
if [ -n "$PR_NUM" ]; then
echo "number=$PR_NUM" >> "$GITHUB_OUTPUT"
else
# Try open PR for this branch
NUM=$(gh pr list --repo "${{ github.repository }}" \
--head "$BRANCH" --state open \
--json number --jq '.[0].number // empty')
# Fallback: extract from sno-qa-<number> branch name
if [ -z "$NUM" ]; then
NUM=$(echo "$BRANCH" | sed -n 's/^sno-qa-\([0-9]\+\)$/\1/p')
fi
echo "number=${NUM}" >> "$GITHUB_OUTPUT"
fi
- name: Run PR analysis
if: steps.pr.outputs.number
- name: Run analysis
if: needs.resolve-matrix.outputs.number
env:
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
mkdir -p qa-guides
pnpm exec tsx scripts/qa-analyze-pr.ts \
--pr-number "${{ steps.pr.outputs.number }}" \
--pr-number "${{ needs.resolve-matrix.outputs.number }}" \
--repo "${{ github.repository }}" \
--output-dir qa-guides
--output-dir qa-guides \
--type "${{ needs.resolve-matrix.outputs.target_type }}"
- name: Upload QA guides
if: always() && steps.pr.outputs.number
if: always() && needs.resolve-matrix.outputs.number
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v6.2.0
with:
name: qa-guides-${{ github.run_id }}
@@ -190,6 +222,7 @@ jobs:
mkdir -p "$QA_ARTIFACTS"
- name: Get PR diff
if: needs.resolve-matrix.outputs.target_type == 'pr'
shell: bash
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -234,19 +267,31 @@ jobs:
-H 'Content-Type: application/json' \
-d '{"Comfy.TutorialCompleted":true}' || echo "Settings pre-seed skipped"
- name: Run BEFORE QA recording
- name: Run QA recording
shell: bash
env:
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
TARGET_TYPE: ${{ needs.resolve-matrix.outputs.target_type }}
run: |
MODE="before"
if [ "$TARGET_TYPE" = "issue" ]; then
MODE="reproduce"
fi
QA_GUIDE_FLAG=""
if [ -f qa-guides/qa-guide-before.json ]; then
echo "Using QA guide for before recording"
echo "Using QA guide for $MODE recording"
QA_GUIDE_FLAG="--qa-guide qa-guides/qa-guide-before.json"
fi
DIFF_FLAG=""
if [ -f "${{ runner.temp }}/pr-diff.txt" ]; then
DIFF_FLAG="--diff ${{ runner.temp }}/pr-diff.txt"
fi
pnpm exec tsx scripts/qa-record.ts \
--mode before \
--diff "${{ runner.temp }}/pr-diff.txt" \
--mode "$MODE" \
$DIFF_FLAG \
--output-dir "$QA_ARTIFACTS" \
--url http://127.0.0.1:8188 \
--test-plan .claude/skills/comfy-qa/SKILL.md \
@@ -276,7 +321,8 @@ jobs:
if: >-
always() &&
needs.resolve-matrix.outputs.skip != 'true' &&
needs.resolve-matrix.result == 'success'
needs.resolve-matrix.result == 'success' &&
needs.resolve-matrix.outputs.target_type == 'pr'
strategy:
fail-fast: false
matrix:
@@ -295,6 +341,21 @@ jobs:
ref: ${{ github.head_ref || github.ref }}
token: ${{ secrets.GITHUB_TOKEN }}
# When triggered via sno-qa-* push, the checkout above gets sno-skills
# (the scripts branch), not the actual PR. Fetch the PR ref and check it out.
- name: Checkout PR head for sno-qa-* triggers
if: >-
!github.head_ref &&
needs.resolve-matrix.outputs.target_type == 'pr' &&
needs.resolve-matrix.outputs.number
shell: bash
env:
PR_NUM: ${{ needs.resolve-matrix.outputs.number }}
run: |
git fetch origin "refs/pull/${PR_NUM}/head"
git checkout FETCH_HEAD
echo "Checked out PR #${PR_NUM} at $(git rev-parse --short HEAD)"
- name: Setup frontend (PR branch)
uses: ./.github/actions/setup-frontend
with:
@@ -401,27 +462,41 @@ jobs:
(github.event.pull_request.number || github.event_name == 'push' || github.event_name == 'workflow_dispatch')
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
issues: write
steps:
- name: Resolve PR number
- name: Configure git identity
run: |
git config --global user.name "github-actions[bot]"
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
- name: Resolve target number and type
id: pr
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUM: ${{ github.event.pull_request.number }}
BRANCH: ${{ github.ref_name }}
REPO: ${{ github.repository }}
run: |
if [ -n "$PR_NUM" ]; then
echo "number=$PR_NUM" >> "$GITHUB_OUTPUT"
NUM="$PR_NUM"
else
# Push event: look up open PR for this branch
NUM=$(gh pr list --repo "${{ github.repository }}" \
NUM=$(gh pr list --repo "$REPO" \
--head "$BRANCH" --state open \
--json number --jq '.[0].number // empty')
# Fallback: extract from sno-qa-<number> branch name
if [ -z "$NUM" ]; then
NUM=$(echo "$BRANCH" | sed -n 's/^sno-qa-\([0-9]\+\)$/\1/p')
fi
echo "number=${NUM}" >> "$GITHUB_OUTPUT"
fi
echo "number=${NUM}" >> "$GITHUB_OUTPUT"
if [ -n "$NUM" ]; then
if gh pr view "$NUM" --repo "$REPO" --json number >/dev/null 2>&1; then
echo "target_type=pr" >> "$GITHUB_OUTPUT"
else
echo "target_type=issue" >> "$GITHUB_OUTPUT"
fi
fi
- name: Checkout repository
@@ -438,6 +513,7 @@ jobs:
pattern: qa-before-*
- name: Download AFTER artifacts
if: needs.qa-after.result == 'success'
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
with:
path: qa-artifacts/after
@@ -521,28 +597,46 @@ jobs:
fi
done
- name: Build PR context for video review
- name: Build context for video review
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TARGET_NUM: ${{ steps.pr.outputs.number }}
TARGET_TYPE: ${{ steps.pr.outputs.target_type }}
REPO: ${{ github.repository }}
run: |
PR_NUM="${{ github.event.pull_request.number || '' }}"
if [ -n "$PR_NUM" ]; then
if [ -z "$TARGET_NUM" ]; then
echo "No target number available, skipping context"
exit 0
fi
if [ "$TARGET_TYPE" = "issue" ]; then
{
echo "### PR #${PR_NUM}"
gh pr view "$PR_NUM" --repo "${{ github.repository }}" \
echo "### Issue #${TARGET_NUM}"
gh issue view "$TARGET_NUM" --repo "$REPO" \
--json title,body --jq '"Title: \(.title)\n\nDescription:\n\(.body)"' 2>/dev/null || true
echo ""
echo "### Comments"
gh api "repos/${REPO}/issues/${TARGET_NUM}/comments" \
--jq '.[].body' 2>/dev/null | head -200 || true
echo ""
echo "This video attempts to reproduce a reported bug on the main branch."
} > pr-context.txt
echo "Issue context saved ($(wc -l < pr-context.txt) lines)"
else
{
echo "### PR #${TARGET_NUM}"
gh pr view "$TARGET_NUM" --repo "$REPO" \
--json title,body --jq '"Title: \(.title)\n\nDescription:\n\(.body)"' 2>/dev/null || true
echo ""
echo "### Changed files"
gh pr diff "$PR_NUM" --repo "${{ github.repository }}" 2>/dev/null \
gh pr diff "$TARGET_NUM" --repo "$REPO" 2>/dev/null \
| grep '^diff --git' | sed 's|diff --git a/||;s| b/.*||' | sort -u || true
echo ""
echo "### Diff (truncated to 300 lines)"
gh pr diff "$PR_NUM" --repo "${{ github.repository }}" 2>/dev/null \
gh pr diff "$TARGET_NUM" --repo "$REPO" 2>/dev/null \
| head -300 || true
} > pr-context.txt
echo "PR context saved ($(wc -l < pr-context.txt) lines)"
else
echo "No PR number available, skipping PR context"
fi
- name: Run video review
@@ -571,7 +665,7 @@ jobs:
done
- name: Generate regression test from QA report
if: needs.resolve-matrix.outputs.mode == 'focused'
if: needs.resolve-matrix.outputs.mode == 'focused' && steps.pr.outputs.target_type == 'pr'
env:
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -626,6 +720,11 @@ jobs:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
RAW_BRANCH: ${{ github.head_ref || github.ref_name }}
BEFORE_SHA: ${{ needs.resolve-matrix.outputs.before_sha }}
AFTER_SHA: ${{ needs.resolve-matrix.outputs.after_sha }}
TARGET_NUM: ${{ steps.pr.outputs.number }}
TARGET_TYPE: ${{ steps.pr.outputs.target_type }}
REPO: ${{ github.repository }}
run: |
npm install -g wrangler@4.74.0 >/dev/null 2>&1
@@ -685,6 +784,21 @@ jobs:
CARD_COUNT=$((CARD_COUNT + 1))
done
# Build commit info for the report header
COMMIT_HTML=""
REPO_URL="https://github.com/${REPO}"
if [ -n "$BEFORE_SHA" ]; then
SHORT_BEFORE="${BEFORE_SHA:0:7}"
COMMIT_HTML="<a href=${REPO_URL}/commit/${BEFORE_SHA} class=sha title='main branch'>main @ ${SHORT_BEFORE}</a>"
fi
if [ -n "$AFTER_SHA" ]; then
SHORT_AFTER="${AFTER_SHA:0:7}"
AFTER_LABEL="PR"
[ -n "$TARGET_NUM" ] && AFTER_LABEL="#${TARGET_NUM}"
COMMIT_HTML="${COMMIT_HTML:+${COMMIT_HTML} &middot; }<a href=${REPO_URL}/commit/${AFTER_SHA} class=sha title='PR head commit'>${AFTER_LABEL} @ ${SHORT_AFTER}</a>"
fi
[ -n "$COMMIT_HTML" ] && COMMIT_HTML=" &middot; ${COMMIT_HTML}"
cat > "$DEPLOY_DIR/index.html" <<INDEXEOF
<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>QA Session Recordings</title>
<link rel=preconnect href=https://fonts.googleapis.com><link rel=preconnect href=https://fonts.gstatic.com crossorigin><link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap" rel=stylesheet>
@@ -747,8 +861,10 @@ jobs:
.reveal{animation:fade-up var(--dur-slow) var(--ease-out) both;animation-delay:calc(var(--i,0) * 120ms)}
@media(prefers-reduced-motion:reduce){.reveal{animation:none}}
@media(max-width:480px){.grid{grid-template-columns:1fr}.card-body{flex-wrap:wrap;gap:.5rem}}
.sha{color:var(--primary);text-decoration:none;font-family:var(--font-mono);font-size:.75rem;font-weight:500;padding:.1rem .4rem;border-radius:.25rem;background:oklch(62% 0.21 265/.08);border:1px solid oklch(62% 0.21 265/.15);transition:all var(--dur-base) var(--ease-out)}
.sha:hover{background:oklch(62% 0.21 265/.15);border-color:var(--primary)}
</style></head><body><div class=container>
<header><div class=header-icon><svg width=20 height=20 viewBox="0 0 24 24" fill=none stroke=currentColor stroke-width=2 stroke-linecap=round stroke-linejoin=round><polygon points="23 7 16 12 23 17 23 7"/><rect x=1 y=5 width=15 height=14 rx=2 ry=2/></svg></div><div><h1>QA Session Recordings</h1><div class=meta>ComfyUI Frontend &middot; Automated QA</div></div></header>
<header><div class=header-icon><svg width=20 height=20 viewBox="0 0 24 24" fill=none stroke=currentColor stroke-width=2 stroke-linecap=round stroke-linejoin=round><polygon points="23 7 16 12 23 17 23 7"/><rect x=1 y=5 width=15 height=14 rx=2 ry=2/></svg></div><div><h1>QA Session Recordings</h1><div class=meta>ComfyUI Frontend &middot; Automated QA${COMMIT_HTML}</div></div></header>
<div class=grid>${CARDS}</div>
</div><script>document.querySelectorAll('[data-md]').forEach(el=>{const t=el.textContent;el.removeAttribute('data-md');el.innerHTML=marked.parse(t)})</script></body></html>
INDEXEOF
@@ -769,17 +885,32 @@ jobs:
echo "url=${URL:-https://${BRANCH}.comfy-qa.pages.dev}" >> "$GITHUB_OUTPUT"
echo "Deployed to: ${URL}"
- name: Post unified QA comment on PR
- name: Post unified QA comment
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VIDEO_BASE: ${{ steps.deploy-videos.outputs.url }}
QA_MODE: ${{ needs.resolve-matrix.outputs.mode }}
TARGET_TYPE: ${{ steps.pr.outputs.target_type }}
BEFORE_SHA: ${{ needs.resolve-matrix.outputs.before_sha }}
AFTER_SHA: ${{ needs.resolve-matrix.outputs.after_sha }}
REPO: ${{ github.repository }}
run: |
RUN="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
RUN="https://github.com/${REPO}/actions/runs/${{ github.run_id }}"
COMMENT_MARKER="<!-- QA_REPORT_COMMENT -->"
MODE_BADGE="🔍 Focused"
if [ "$QA_MODE" = "full" ]; then MODE_BADGE="🔬 Full (3-OS)"; fi
if [ "$TARGET_TYPE" = "issue" ]; then MODE_BADGE="🐛 Issue Reproduce"; fi
# Build commit links
COMMIT_LINE=""
REPO_URL="https://github.com/${REPO}"
if [ -n "$BEFORE_SHA" ] || [ -n "$AFTER_SHA" ]; then
PARTS=""
[ -n "$BEFORE_SHA" ] && PARTS="main [\`${BEFORE_SHA:0:7}\`](${REPO_URL}/commit/${BEFORE_SHA})"
[ -n "$AFTER_SHA" ] && PARTS="${PARTS:+${PARTS} · }PR [\`${AFTER_SHA:0:7}\`](${REPO_URL}/commit/${AFTER_SHA})"
COMMIT_LINE="**Commits**: ${PARTS}"
fi
# Build video section with GIF thumbnails linking to full videos
VIDEO_SECTION=""
@@ -825,7 +956,8 @@ jobs:
${VIDEO_SECTION}
**Run**: [${RUN}](${RUN}) · [Download artifacts](${RUN}#artifacts) · [All videos](${VIDEO_BASE})
${VIDEO_REVIEW_SECTION}
${COMMIT_LINE:+${COMMIT_LINE}
}${VIDEO_REVIEW_SECTION}
EOF
)
@@ -841,6 +973,9 @@ jobs:
if [ -n "$EXISTING" ]; then
gh api --method PATCH "repos/${{ github.repository }}/issues/comments/${EXISTING}" \
--field body="$BODY"
elif [ "$TARGET_TYPE" = "issue" ]; then
gh issue comment "$PR_NUM" \
--repo ${{ github.repository }} --body "$BODY"
else
gh pr comment "$PR_NUM" \
--repo ${{ github.repository }} --body "$BODY"

244
scripts/qa-analyze-pr.ts
View File

@@ -49,6 +49,8 @@ interface PrThread {
diff: string
}
type TargetType = 'pr' | 'issue'
interface Options {
prNumber: string
repo: string
@@ -57,6 +59,7 @@ interface Options {
apiKey: string
mediaBudgetBytes: number
maxVideoBytes: number
type: TargetType
}
// ── CLI parsing ──
@@ -67,7 +70,8 @@ function parseArgs(): Options {
model: 'gemini-2.5-pro',
apiKey: process.env.GEMINI_API_KEY || '',
mediaBudgetBytes: 20 * 1024 * 1024,
maxVideoBytes: 10 * 1024 * 1024
maxVideoBytes: 10 * 1024 * 1024,
type: 'pr'
}
for (let i = 0; i < args.length; i++) {
@@ -84,9 +88,12 @@ function parseArgs(): Options {
case '--model':
opts.model = args[++i]
break
case '--type':
opts.type = args[++i] as TargetType
break
case '--help':
console.warn(
'Usage: qa-analyze-pr.ts --pr-number <num> --repo <owner/repo> --output-dir <path> [--model <model>]'
'Usage: qa-analyze-pr.ts --pr-number <num> --repo <owner/repo> --output-dir <path> [--model <model>] [--type pr|issue]'
)
process.exit(0)
}
@@ -177,6 +184,43 @@ function fetchPrThread(prNumber: string, repo: string): PrThread {
}
}
interface IssueThread {
title: string
body: string
labels: string[]
comments: string[]
}
function fetchIssueThread(issueNumber: string, repo: string): IssueThread {
console.warn('Fetching issue thread...')
const issueView = ghExec(
`gh issue view ${issueNumber} --repo ${repo} --json title,body,labels`
)
const issueData = issueView
? JSON.parse(issueView)
: { title: '', body: '', labels: [] }
const commentsRaw = ghExec(
`gh api repos/${repo}/issues/${issueNumber}/comments --paginate`
)
const comments: string[] = commentsRaw
? JSON.parse(commentsRaw).map((c: { body: string }) => c.body)
: []
console.warn(
`Issue #${issueNumber}: "${issueData.title}" | ` +
`${comments.length} comments`
)
return {
title: issueData.title || '',
body: issueData.body || '',
labels: (issueData.labels || []).map((l: { name: string }) => l.name),
comments
}
}
// ── Media extraction ──
const MEDIA_EXTENSIONS = /\.(png|jpg|jpeg|gif|webp|mp4|webm|mov)$/i
@@ -217,6 +261,26 @@ export function extractMediaUrls(text: string): string[] {
// ── Media downloading ──
const ALLOWED_MEDIA_DOMAINS = [
'github.com',
'raw.githubusercontent.com',
'user-images.githubusercontent.com',
'private-user-images.githubusercontent.com',
'objects.githubusercontent.com',
'github.githubassets.com'
]
function isAllowedMediaDomain(url: string): boolean {
try {
const hostname = new URL(url).hostname
return ALLOWED_MEDIA_DOMAINS.some(
(domain) => hostname === domain || hostname.endsWith(`.${domain}`)
)
} catch {
return false
}
}
async function downloadMedia(
urls: string[],
outputDir: string,
@@ -237,10 +301,16 @@ async function downloadMedia(
break
}
if (!isAllowedMediaDomain(url)) {
console.warn(`Skipping non-GitHub URL: ${url.slice(0, 80)}`)
continue
}
try {
const response = await fetch(url, {
signal: AbortSignal.timeout(15_000),
headers: { Accept: 'image/*,video/*' }
headers: { Accept: 'image/*,video/*' },
redirect: 'follow'
})
if (!response.ok) {
@@ -248,6 +318,17 @@ async function downloadMedia(
continue
}
const contentLength = response.headers.get('content-length')
if (contentLength) {
const declaredSize = Number.parseInt(contentLength, 10)
if (declaredSize > budgetBytes - totalBytes) {
console.warn(
`Content-Length ${declaredSize} would exceed budget, skipping ${url}`
)
continue
}
}
const contentType = response.headers.get('content-type') || ''
const buffer = Buffer.from(await response.arrayBuffer())
@@ -321,6 +402,63 @@ function guessMimeType(ext: string): string {
// ── Gemini analysis ──
function buildIssueAnalysisPrompt(issue: IssueThread): string {
const allText = [
`# Issue: ${issue.title}`,
'',
'## Description',
issue.body,
'',
issue.comments.length > 0
? `## Comments\n${issue.comments.join('\n\n---\n\n')}`
: ''
]
.filter(Boolean)
.join('\n')
return `You are a senior QA engineer analyzing a bug report for ComfyUI frontend (a Vue 3 + TypeScript web application for AI image generation workflows).
Your task: Generate a single targeted QA reproduction guide to verify this bug on the current main branch.
${allText}
## Available test actions
Each step must use one of these actions:
- "openMenu" — clicks the Comfy hamburger menu (top-left C logo)
- "hoverMenuItem" — hovers a top-level menu item to open submenu (label required)
- "clickMenuItem" — clicks an item in the visible submenu (label required)
- "fillDialog" — fills dialog input and presses Enter (text required)
- "pressKey" — presses a keyboard key (key required)
- "click" — clicks an element by visible text (text required)
- "wait" — waits briefly (ms required, max 3000)
- "screenshot" — takes a screenshot (name required)
## Output format
Return a JSON object with exactly one key: "reproduce", containing:
{
"summary": "One sentence: what bug this issue reports",
"test_focus": "Specific behavior to reproduce",
"prerequisites": ["e.g. Load default workflow"],
"steps": [
{
"action": "openMenu",
"description": "Open the main menu to trigger the reported bug",
"expected_before": "What should happen if the bug is present"
}
],
"visual_checks": ["Specific visual evidence of the bug to look for"]
}
## Rules
- REPRODUCE guide: 3-6 steps, under 30 seconds. Follow the issue's reproduction steps closely.
- Focus on triggering and demonstrating the SPECIFIC bug reported.
- Use information from the issue description and comments to understand the bug.
- Include at least one screenshot step to capture the bug state.
- Do NOT include login steps.
- Menu pattern: openMenu -> hoverMenuItem -> clickMenuItem or screenshot.
- Output ONLY valid JSON, no markdown fences or explanation.`
}
function buildAnalysisPrompt(thread: PrThread): string {
const allText = [
`# PR: ${thread.title}`,
@@ -479,16 +617,109 @@ async function analyzeWithGemini(
return { before, after }
}
async function analyzeIssueWithGemini(
issue: IssueThread,
media: Array<{ path: string; mimeType: string }>,
model: string,
apiKey: string
): Promise<QaGuide> {
const genAI = new GoogleGenerativeAI(apiKey)
const geminiModel = genAI.getGenerativeModel({ model })
const prompt = buildIssueAnalysisPrompt(issue)
const parts: Array<
{ text: string } | { inlineData: { mimeType: string; data: string } }
> = [{ text: prompt }]
for (const item of media) {
try {
const buffer = readFileSync(item.path)
parts.push({
inlineData: {
mimeType: item.mimeType,
data: buffer.toString('base64')
}
})
} catch (err) {
console.warn(
`Failed to read media ${item.path}: ${(err as Error).message}`
)
}
}
console.warn(
`Sending to ${model}: ${prompt.length} chars text, ${media.length} media files`
)
const result = await geminiModel.generateContent({
contents: [{ role: 'user', parts }],
generationConfig: {
temperature: 0.2,
maxOutputTokens: 8192,
responseMimeType: 'application/json'
}
})
let text = result.response.text()
text = text
.replace(/^```(?:json)?\n?/gm, '')
.replace(/```$/gm, '')
.trim()
console.warn('Gemini response received')
console.warn('Raw response (first 500 chars):', text.slice(0, 500))
const parsed = JSON.parse(text)
const guide: QaGuide =
parsed.reproduce ?? parsed.qa_guide?.reproduce ?? parsed
return guide
}
// ── Main ──
async function main() {
const opts = parseArgs()
mkdirSync(opts.outputDir, { recursive: true })
// A. Fetch PR thread
if (opts.type === 'issue') {
await analyzeIssue(opts)
} else {
await analyzePr(opts)
}
}
async function analyzeIssue(opts: Options) {
const issue = fetchIssueThread(opts.prNumber, opts.repo)
const allText = [issue.body, ...issue.comments].join('\n')
const mediaUrls = extractMediaUrls(allText)
console.warn(`Found ${mediaUrls.length} media URLs`)
const media = await downloadMedia(
mediaUrls,
opts.outputDir,
opts.mediaBudgetBytes,
opts.maxVideoBytes
)
const guide = await analyzeIssueWithGemini(
issue,
media,
opts.model,
opts.apiKey
)
const beforePath = resolve(opts.outputDir, 'qa-guide-before.json')
writeFileSync(beforePath, JSON.stringify(guide, null, 2))
console.warn(`Wrote QA guide:`)
console.warn(` Reproduce: ${beforePath}`)
}
async function analyzePr(opts: Options) {
const thread = fetchPrThread(opts.prNumber, opts.repo)
// B. Extract media URLs from all text
const allText = [
thread.body,
...thread.issueComments,
@@ -498,7 +729,6 @@ async function main() {
const mediaUrls = extractMediaUrls(allText)
console.warn(`Found ${mediaUrls.length} media URLs`)
// C. Download media
const media = await downloadMedia(
mediaUrls,
opts.outputDir,
@@ -506,10 +736,8 @@ async function main() {
opts.maxVideoBytes
)
// D. Send to Gemini
const guides = await analyzeWithGemini(thread, media, opts.model, opts.apiKey)
// E. Write guides
const beforePath = resolve(opts.outputDir, 'qa-guide-before.json')
const afterPath = resolve(opts.outputDir, 'qa-guide-after.json')
writeFileSync(beforePath, JSON.stringify(guides.before, null, 2))

175
scripts/qa-batch.sh Executable file
View File

@@ -0,0 +1,175 @@
#!/usr/bin/env bash
# Batch-trigger QA runs by creating and pushing sno-qa-* branches.
#
# Usage:
# ./scripts/qa-batch.sh 10394 10238 9996 # Trigger specific numbers
# ./scripts/qa-batch.sh --from tmp/issues.md --top 5 # From triage file
# ./scripts/qa-batch.sh --dry-run 10394 10238 # Preview only
# ./scripts/qa-batch.sh --cleanup # Delete old sno-qa-* branches
set -euo pipefail
DELAY=5
DRY_RUN=false
CLEANUP=false
FROM_FILE=""
TOP_N=0
NUMBERS=()
die() { echo "error: $*" >&2; exit 1; }
usage() {
cat <<'EOF'
Usage: qa-batch.sh [options] [numbers...]
Options:
--from <file> Extract numbers from a triage markdown file
--top <N> Take first N entries from Tier 1 (requires --from)
--dry-run Print what would happen without pushing
--cleanup Delete all sno-qa-* remote branches
--delay <secs> Seconds between pushes (default: 5)
-h, --help Show this help
EOF
exit 0
}
# --- Parse args ---
while [[ $# -gt 0 ]]; do
case "$1" in
--from) FROM_FILE="$2"; shift 2 ;;
--top) TOP_N="$2"; shift 2 ;;
--dry-run) DRY_RUN=true; shift ;;
--cleanup) CLEANUP=true; shift ;;
--delay) DELAY="$2"; shift 2 ;;
-h|--help) usage ;;
-*) die "unknown option: $1" ;;
*) NUMBERS+=("$1"); shift ;;
esac
done
# --- Cleanup mode ---
if $CLEANUP; then
echo "Fetching remote sno-qa-* branches..."
branches=$(git ls-remote --heads origin 'refs/heads/sno-qa-*' | awk '{print $2}' | sed 's|refs/heads/||')
if [[ -z "$branches" ]]; then
echo "No sno-qa-* branches found on remote."
exit 0
fi
echo "Found branches:"
echo "$branches" | sed 's/^/ /'
echo
if $DRY_RUN; then
echo "[dry-run] Would delete the above branches."
exit 0
fi
read -rp "Delete all of the above? [y/N] " confirm
if [[ "$confirm" != "y" && "$confirm" != "Y" ]]; then
echo "Aborted."
exit 0
fi
for branch in $branches; do
echo "Deleting origin/$branch..."
git push origin --delete "$branch"
done
echo "Done. Cleaned up $(echo "$branches" | wc -l | tr -d ' ') branches."
exit 0
fi
# --- Extract numbers from markdown ---
if [[ -n "$FROM_FILE" ]]; then
[[ -f "$FROM_FILE" ]] || die "file not found: $FROM_FILE"
[[ "$TOP_N" -gt 0 ]] || die "--top N required with --from"
# Extract Tier 1 table rows: | N | [#NNNNN](...) | ...
# Stop at the next ## heading after Tier 1
extracted=$(awk '/^## Tier 1/,/^## Tier [^1]/' "$FROM_FILE" \
| grep -oP '\[#\K\d+' \
| head -n "$TOP_N")
if [[ -z "$extracted" ]]; then
die "no numbers found in $FROM_FILE"
fi
while IFS= read -r num; do
NUMBERS+=("$num")
done <<< "$extracted"
fi
[[ ${#NUMBERS[@]} -gt 0 ]] || die "no numbers specified. Use positional args or --from/--top."
# --- Validate ---
for num in "${NUMBERS[@]}"; do
[[ "$num" =~ ^[0-9]+$ ]] || die "invalid number: $num"
done
# Deduplicate
NUMBERS=($(printf '%s\n' "${NUMBERS[@]}" | sort -un))
# --- Push branches ---
echo "Triggering QA for: ${NUMBERS[*]}"
if $DRY_RUN; then
echo "[dry-run]"
fi
echo
pushed=()
skipped=()
# Fetch remote refs once
remote_refs=$(git ls-remote --heads origin 'refs/heads/sno-qa-*' 2>/dev/null | awk '{print $2}' | sed 's|refs/heads/||')
for num in "${NUMBERS[@]}"; do
branch="sno-qa-$num"
# Check if already exists on remote
if echo "$remote_refs" | grep -qx "$branch"; then
echo " skip: $branch (already exists on remote)"
skipped+=("$num")
continue
fi
if $DRY_RUN; then
echo " would push: $branch"
pushed+=("$num")
continue
fi
# Create branch at current HEAD and push
git branch -f "$branch" HEAD
git push origin "$branch"
pushed+=("$num")
echo " pushed: $branch"
# Clean up local branch
git branch -D "$branch" 2>/dev/null || true
# Delay between pushes to avoid CI concurrency storm
if [[ "$num" != "${NUMBERS[-1]}" ]]; then
echo " waiting ${DELAY}s..."
sleep "$DELAY"
fi
done
# --- Summary ---
echo
echo "=== Summary ==="
echo "Triggered: ${#pushed[@]}"
echo "Skipped: ${#skipped[@]}"
if [[ ${#pushed[@]} -gt 0 ]]; then
echo
echo "Triggered numbers: ${pushed[*]}"
repo_url=$(git remote get-url origin | sed 's/\.git$//' | sed 's|git@github.com:|https://github.com/|')
echo "Actions: ${repo_url}/actions"
fi
if [[ ${#skipped[@]} -gt 0 ]]; then
echo
echo "Skipped (already exist): ${skipped[*]}"
echo "Use --cleanup first to remove old branches."
fi

80
scripts/qa-record.ts
View File

@@ -19,7 +19,7 @@
import { chromium } from '@playwright/test'
import type { Page } from '@playwright/test'
import { GoogleGenerativeAI } from '@google/generative-ai'
import { readFileSync, mkdirSync, readdirSync, renameSync } from 'fs'
import { readFileSync, mkdirSync, readdirSync, renameSync, statSync } from 'fs'
// ── Types ──
@@ -33,9 +33,11 @@ type TestAction =
| { action: 'wait'; ms: number }
| { action: 'screenshot'; name: string }
type RecordMode = 'before' | 'after' | 'reproduce'
interface Options {
mode: 'before' | 'after'
diffFile: string
mode: RecordMode
diffFile?: string
outputDir: string
serverUrl: string
model: string
@@ -57,7 +59,7 @@ function parseArgs(): Options {
for (let i = 0; i < args.length; i++) {
switch (args[i]) {
case '--mode':
opts.mode = args[++i] as 'before' | 'after'
opts.mode = args[++i] as RecordMode
break
case '--diff':
opts.diffFile = args[++i]
@@ -79,19 +81,30 @@ function parseArgs(): Options {
break
case '--help':
console.warn(
'Usage: qa-record.ts --mode before|after --diff <path> --output-dir <path> [--url <url>] [--model <model>] [--test-plan <path>] [--qa-guide <path>]'
'Usage: qa-record.ts --mode before|after|reproduce --output-dir <path> [--diff <path>] [--url <url>] [--model <model>] [--test-plan <path>] [--qa-guide <path>]'
)
process.exit(0)
}
}
if (!opts.mode || !opts.diffFile || !opts.outputDir) {
if (!opts.mode || !opts.outputDir) {
console.error('Required: --mode before|after|reproduce --output-dir <path>')
process.exit(1)
}
const validModes: RecordMode[] = ['before', 'after', 'reproduce']
if (!validModes.includes(opts.mode)) {
console.error(
'Required: --mode before|after --diff <path> --output-dir <path>'
`Invalid --mode "${opts.mode}". Must be one of: ${validModes.join(', ')}`
)
process.exit(1)
}
if (!opts.diffFile && opts.mode !== 'reproduce') {
console.error('--diff is required for before/after modes')
process.exit(1)
}
if (!opts.apiKey) {
console.error('GEMINI_API_KEY environment variable is required')
process.exit(1)
@@ -108,10 +121,15 @@ function buildPrompt(
testPlan?: string,
qaGuide?: string
): string {
const modeDesc =
mode === 'before'
? 'BEFORE (main branch). Show the OLD state briefly — under 15 seconds. One quick demonstration of missing feature / old behavior.'
: 'AFTER (PR branch). Prove the changes work — 3-6 targeted steps, under 30 seconds.'
const modeDescriptions: Record<string, string> = {
before:
'BEFORE (main branch). Show the OLD state briefly — under 15 seconds. One quick demonstration of missing feature / old behavior.',
after:
'AFTER (PR branch). Prove the changes work — 3-6 targeted steps, under 30 seconds.',
reproduce:
'REPRODUCE a reported issue on main branch. Follow the reproduction steps to trigger and demonstrate the reported bug — 3-6 steps, under 30 seconds.'
}
const modeDesc = modeDescriptions[mode] ?? modeDescriptions.before
const qaGuideSection = qaGuide
? `
@@ -149,18 +167,15 @@ Each step is an object with an "action" field:
- { "action": "wait", "ms": 1000 } — waits (use sparingly, max 3000ms)
- { "action": "screenshot", "name": "step-name" } — takes a screenshot
${qaGuideSection}${testPlanSection}
## PR Diff
\`\`\`
${diff.slice(0, 3000)}
\`\`\`
${diff ? `## PR Diff\n\`\`\`\n${diff.slice(0, 3000)}\n\`\`\`` : ''}
## Rules
- Output ONLY a valid JSON array of actions, no markdown fences or explanation
- ${mode === 'before' ? 'Keep it minimal — just show the old/missing behavior' : 'Test the specific behavior that changed in the PR'}
- ${mode === 'reproduce' ? 'Follow the reproduction steps to trigger and demonstrate the reported bug' : mode === 'before' ? 'Keep it minimal — just show the old/missing behavior' : 'Test the specific behavior that changed in the PR'}
- Always include at least one screenshot
- Do NOT include login steps (handled automatically)
- Menu navigation pattern: openMenu → hoverMenuItem → clickMenuItem (or screenshot)
${qaGuide ? '- Follow the QA Analysis Guide steps closely — they are PR-specific and well-researched' : '- Pick test steps from the QA test plan categories that are most relevant to the diff'}
${qaGuide ? '- Follow the QA Analysis Guide steps closely — they are well-researched and specific' : diff ? '- Pick test steps from the QA test plan categories that are most relevant to the diff' : '- Pick test steps from the QA test plan categories most likely to reveal bugs'}
## Example output
[
@@ -176,7 +191,7 @@ ${qaGuide ? '- Follow the QA Analysis Guide steps closely — they are PR-specif
}
async function generateTestSteps(opts: Options): Promise<TestAction[]> {
const diff = readFileSync(opts.diffFile, 'utf-8')
const diff = opts.diffFile ? readFileSync(opts.diffFile, 'utf-8') : ''
const testPlan = opts.testPlanFile
? readFileSync(opts.testPlanFile, 'utf-8')
: undefined
@@ -241,6 +256,14 @@ const FALLBACK_AFTER: TestAction[] = [
{ action: 'screenshot', name: 'editor-after' }
]
const FALLBACK_REPRODUCE: TestAction[] = FALLBACK_BEFORE
const FALLBACK_STEPS: Record<RecordMode, TestAction[]> = {
before: FALLBACK_BEFORE,
after: FALLBACK_AFTER,
reproduce: FALLBACK_REPRODUCE
}
// ── Playwright helpers ──
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
@@ -372,11 +395,13 @@ async function executeSteps(
case 'wait':
await sleep(Math.min(step.ms, 5000))
break
case 'screenshot':
case 'screenshot': {
const safeName = step.name.replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 100)
await page.screenshot({
path: `${outputDir}/${step.name}.png`
path: `${outputDir}/${safeName}.png`
})
break
}
default:
console.warn(`Unknown action: ${JSON.stringify(step)}`)
}
@@ -485,7 +510,7 @@ async function main() {
steps = await generateTestSteps(opts)
} catch (err) {
console.warn('Gemini generation failed, using fallback steps:', err)
steps = opts.mode === 'before' ? FALLBACK_BEFORE : FALLBACK_AFTER
steps = FALLBACK_STEPS[opts.mode]
}
// Launch browser with video recording (Chromium for WebGL support)
@@ -524,12 +549,17 @@ async function main() {
// Rename the recorded video to expected filename
const videoName =
opts.mode === 'before' ? 'qa-before-session.webm' : 'qa-session.webm'
// reproduce mode uses 'qa-session.webm' (same as after — it's the primary video)
const knownNames = new Set(['qa-before-session.webm', 'qa-session.webm'])
const files = readdirSync(opts.outputDir).filter(
(f) => f.endsWith('.webm') && !knownNames.has(f)
)
const files = readdirSync(opts.outputDir)
.filter((f) => f.endsWith('.webm') && !knownNames.has(f))
.sort((a, b) => {
const mtimeA = statSync(`${opts.outputDir}/${a}`).mtimeMs
const mtimeB = statSync(`${opts.outputDir}/${b}`).mtimeMs
return mtimeB - mtimeA
})
if (files.length > 0) {
const recorded = files[files.length - 1]
const recorded = files[0]
renameSync(
`${opts.outputDir}/${recorded}`,
`${opts.outputDir}/${videoName}`

76
scripts/qa-video-review.ts
View File

@@ -399,20 +399,42 @@ function buildSingleVideoPrompt(
''
]
if (prContext) {
lines.push(
'## PR Context',
'The video is a QA session testing a specific pull request.',
'Your review MUST evaluate whether the PR achieves its stated purpose.',
'',
prContext,
'',
'## Review Instructions',
"1. Does the video demonstrate the PR's intended behavior working correctly?",
'2. Are there regressions or side effects caused by the PR changes?',
'3. Does the observed behavior match what the PR claims to implement/fix?',
''
const isIssueContext =
prContext &&
/^### Issue #|^Title:.*\bbug\b|^This video attempts to reproduce/im.test(
prContext
)
if (prContext) {
if (isIssueContext) {
lines.push(
'## Issue Context',
'This video attempts to reproduce a reported bug on the main branch.',
'Your review MUST evaluate whether the reported bug is visible and reproducible.',
'',
prContext,
'',
'## Review Instructions',
'1. Does the video demonstrate the reported bug occurring?',
'2. Is the bug clearly visible and reproducible from the steps shown?',
'3. Are there any other issues visible during the reproduction attempt?',
''
)
} else {
lines.push(
'## PR Context',
'The video is a QA session testing a specific pull request.',
'Your review MUST evaluate whether the PR achieves its stated purpose.',
'',
prContext,
'',
'## Review Instructions',
"1. Does the video demonstrate the PR's intended behavior working correctly?",
'2. Are there regressions or side effects caused by the PR changes?',
'3. Does the observed behavior match what the PR claims to implement/fix?',
''
)
}
}
lines.push(
@@ -426,9 +448,11 @@ function buildSingleVideoPrompt(
'',
'Return markdown with these sections exactly:',
'## Summary',
prContext
? '(Explain what the PR intended and whether the video confirms it works)'
: '',
isIssueContext
? '(Explain what bug was reported and whether the video confirms it is reproducible)'
: prContext
? '(Explain what the PR intended and whether the video confirms it works)'
: '',
'## Confirmed Issues',
'For each confirmed issue, use this exact format (one block per issue):',
'',
@@ -453,6 +477,18 @@ function buildSingleVideoPrompt(
return lines.filter(Boolean).join('\n')
}
const MAX_VIDEO_BYTES = 100 * 1024 * 1024
async function readVideoFile(videoPath: string): Promise<Buffer> {
const fileStat = await stat(videoPath)
if (fileStat.size > MAX_VIDEO_BYTES) {
throw new Error(
`Video ${basename(videoPath)} is ${formatBytes(fileStat.size)}, exceeds ${formatBytes(MAX_VIDEO_BYTES)} limit`
)
}
return readFile(videoPath)
}
async function requestGeminiReview(options: {
apiKey: string
model: string
@@ -478,7 +514,7 @@ async function requestGeminiReview(options: {
> = [{ text: prompt }]
if (isComparative) {
const beforeBuffer = await readFile(options.beforeVideoPath)
const beforeBuffer = await readVideoFile(options.beforeVideoPath)
parts.push(
{ text: 'Video 1 — BEFORE (main branch):' },
{
@@ -490,7 +526,7 @@ async function requestGeminiReview(options: {
)
}
const afterBuffer = await readFile(options.videoPath)
const afterBuffer = await readVideoFile(options.videoPath)
if (isComparative) {
parts.push({ text: 'Video 2 — AFTER (PR branch):' })
}
@@ -501,7 +537,9 @@ async function requestGeminiReview(options: {
}
})
const result = await model.generateContent(parts)
const result = await model.generateContent(parts, {
timeout: options.timeoutMs
})
const response = result.response
const text = response.text()