mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-04-19 22:09:37 +00:00
feat: integrate comfy-qa skill test plan into QA recording pipeline
Pass the comprehensive test plan from .claude/skills/comfy-qa/SKILL.md to Gemini when generating test steps. This gives Gemini knowledge of all 12 QA categories (canvas, menus, sidebar, settings, etc.) so it picks the most relevant tests for each PR. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
6
.github/workflows/pr-qa.yaml
vendored
6
.github/workflows/pr-qa.yaml
vendored
@@ -163,7 +163,8 @@ jobs:
|
||||
--mode before \
|
||||
--diff "${{ runner.temp }}/pr-diff.txt" \
|
||||
--output-dir "$QA_ARTIFACTS" \
|
||||
--url http://127.0.0.1:8188
|
||||
--url http://127.0.0.1:8188 \
|
||||
--test-plan .claude/skills/comfy-qa/SKILL.md
|
||||
|
||||
- name: Stop server after BEFORE run
|
||||
if: needs.resolve-matrix.outputs.mode == 'focused'
|
||||
@@ -200,7 +201,8 @@ jobs:
|
||||
--mode after \
|
||||
--diff "${{ runner.temp }}/pr-diff.txt" \
|
||||
--output-dir "$QA_ARTIFACTS" \
|
||||
--url http://127.0.0.1:8188
|
||||
--url http://127.0.0.1:8188 \
|
||||
--test-plan .claude/skills/comfy-qa/SKILL.md
|
||||
|
||||
- name: Collect artifacts
|
||||
if: always()
|
||||
|
||||
@@ -40,6 +40,7 @@ interface Options {
|
||||
serverUrl: string
|
||||
model: string
|
||||
apiKey: string
|
||||
testPlanFile?: string
|
||||
}
|
||||
|
||||
// ── CLI parsing ──
|
||||
@@ -69,9 +70,12 @@ function parseArgs(): Options {
|
||||
case '--model':
|
||||
opts.model = args[++i]
|
||||
break
|
||||
case '--test-plan':
|
||||
opts.testPlanFile = args[++i]
|
||||
break
|
||||
case '--help':
|
||||
console.warn(
|
||||
'Usage: qa-record.ts --mode before|after --diff <path> --output-dir <path> [--url <url>] [--model <model>]'
|
||||
'Usage: qa-record.ts --mode before|after --diff <path> --output-dir <path> [--url <url>] [--model <model>] [--test-plan <path>]'
|
||||
)
|
||||
process.exit(0)
|
||||
}
|
||||
@@ -94,12 +98,22 @@ function parseArgs(): Options {
|
||||
|
||||
// ── Gemini test step generation ──
|
||||
|
||||
function buildPrompt(mode: string, diff: string): string {
|
||||
function buildPrompt(mode: string, diff: string, testPlan?: string): string {
|
||||
const modeDesc =
|
||||
mode === 'before'
|
||||
? 'BEFORE (main branch). Show the OLD state briefly — under 15 seconds. One quick demonstration of missing feature / old behavior.'
|
||||
: 'AFTER (PR branch). Prove the changes work — 3-6 targeted steps, under 30 seconds.'
|
||||
|
||||
const testPlanSection = testPlan
|
||||
? `
|
||||
## QA Test Plan Reference
|
||||
Use this test plan to identify which test categories are relevant to the PR diff.
|
||||
Pick test steps from the most relevant categories.
|
||||
|
||||
${testPlan.slice(0, 4000)}
|
||||
`
|
||||
: ''
|
||||
|
||||
return `You are generating test steps for a ComfyUI frontend QA recording.
|
||||
|
||||
MODE: ${modeDesc}
|
||||
@@ -114,7 +128,7 @@ Each step is an object with an "action" field:
|
||||
- { "action": "click", "text": "Button Text" } — clicks an element by visible text
|
||||
- { "action": "wait", "ms": 1000 } — waits (use sparingly, max 3000ms)
|
||||
- { "action": "screenshot", "name": "step-name" } — takes a screenshot
|
||||
|
||||
${testPlanSection}
|
||||
## PR Diff
|
||||
\`\`\`
|
||||
${diff.slice(0, 3000)}
|
||||
@@ -126,6 +140,7 @@ ${diff.slice(0, 3000)}
|
||||
- Always include at least one screenshot
|
||||
- Do NOT include login steps (handled automatically)
|
||||
- Menu navigation pattern: openMenu → hoverMenuItem → clickMenuItem (or screenshot)
|
||||
- Pick test steps from the QA test plan categories that are most relevant to the diff
|
||||
|
||||
## Example output
|
||||
[
|
||||
@@ -142,7 +157,10 @@ ${diff.slice(0, 3000)}
|
||||
|
||||
async function generateTestSteps(opts: Options): Promise<TestAction[]> {
|
||||
const diff = readFileSync(opts.diffFile, 'utf-8')
|
||||
const prompt = buildPrompt(opts.mode, diff)
|
||||
const testPlan = opts.testPlanFile
|
||||
? readFileSync(opts.testPlanFile, 'utf-8')
|
||||
: undefined
|
||||
const prompt = buildPrompt(opts.mode, diff, testPlan)
|
||||
|
||||
const genAI = new GoogleGenerativeAI(opts.apiKey)
|
||||
const model = genAI.getGenerativeModel({ model: opts.model })
|
||||
|
||||
Reference in New Issue
Block a user