# Automated QA of ComfyUI frontend using Claude CLI + playwright-cli. # Two modes: # Focused (qa-changes label): Linux-only, tests areas affected by PR changes # Full (qa-full label): 3-OS matrix, full test plan name: 'PR: QA' on: # TODO: remove push trigger before merge push: branches: [sno-skills] pull_request: types: [labeled] branches: [main] workflow_dispatch: inputs: mode: description: 'QA mode' type: choice options: [focused, full] default: focused concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: resolve-matrix: runs-on: ubuntu-latest outputs: os: ${{ steps.set.outputs.os }} mode: ${{ steps.set.outputs.mode }} skip: ${{ steps.set.outputs.skip }} steps: - name: Determine QA mode id: set env: LABEL: ${{ github.event.label.name }} EVENT_ACTION: ${{ github.event.action }} EVENT_NAME: ${{ github.event_name }} INPUT_MODE: ${{ inputs.mode }} run: | FULL=false # Only run on label events if it's one of our labels if [ "$EVENT_ACTION" = "labeled" ] && \ [ "$LABEL" != "qa-changes" ] && [ "$LABEL" != "qa-full" ]; then echo "skip=true" >> "$GITHUB_OUTPUT" fi # TODO: remove push trigger before merge if [ "$EVENT_NAME" = "push" ]; then FULL=true fi # Full QA triggers if [ "$EVENT_NAME" = "workflow_dispatch" ] && \ [ "$INPUT_MODE" = "full" ]; then FULL=true fi if [ "$LABEL" = "qa-full" ]; then FULL=true fi if [ "$FULL" = "true" ]; then echo 'os=["ubuntu-latest","macos-latest","windows-latest"]' >> "$GITHUB_OUTPUT" echo "mode=full" >> "$GITHUB_OUTPUT" else echo 'os=["ubuntu-latest"]' >> "$GITHUB_OUTPUT" echo "mode=focused" >> "$GITHUB_OUTPUT" fi echo "Mode: $([ "$FULL" = "true" ] && echo full || echo focused)" qa: needs: resolve-matrix if: needs.resolve-matrix.outputs.skip != 'true' strategy: fail-fast: false matrix: os: ${{ fromJson(needs.resolve-matrix.outputs.os) }} runs-on: ${{ matrix.os }} timeout-minutes: 60 permissions: pull-requests: write env: QA_MODE: ${{ needs.resolve-matrix.outputs.mode }} steps: - name: Set QA artifacts path shell: bash run: echo "QA_ARTIFACTS=$RUNNER_TEMP/qa-artifacts" >> "$GITHUB_ENV" - name: Checkout repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 ref: ${{ github.head_ref || github.ref }} token: ${{ secrets.GITHUB_TOKEN }} - name: Setup frontend uses: ./.github/actions/setup-frontend with: include_build_step: true - name: Setup and start ComfyUI server uses: ./.github/actions/setup-comfyui-server with: launch_server: 'true' - name: Wait for ComfyUI server shell: bash run: | echo "Waiting for ComfyUI server..." for i in $(seq 1 60); do if curl -sf http://127.0.0.1:8188/api/system_stats >/dev/null 2>&1; then echo "Server ready"; exit 0 fi; sleep 2 done echo "::error::Server timeout"; exit 1 - name: Install playwright-cli and Claude Code shell: bash run: npm install -g @playwright/cli@latest @anthropic-ai/claude-code@2.1.71 - name: Get PR diff for focused QA if: needs.resolve-matrix.outputs.mode == 'focused' shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh pr diff ${{ github.event.pull_request.number || '' }} \ --repo ${{ github.repository }} > "${{ runner.temp }}/pr-diff.txt" 2>/dev/null || \ git diff origin/main...HEAD > "${{ runner.temp }}/pr-diff.txt" # Summarize changed files for the prompt echo "Changed files:" grep '^diff --git' "${{ runner.temp }}/pr-diff.txt" | \ sed 's|diff --git a/||;s| b/.*||' | sort -u | tee "${{ runner.temp }}/changed-files.txt" - name: Write QA prompt shell: bash env: BRANCH: ${{ github.head_ref || github.ref_name }} PR_NUM: ${{ github.event.pull_request.number || 'N/A' }} SHA: ${{ github.sha }} run: | OS_LOWER=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]') if [ "$QA_MODE" = "full" ]; then cat > "${{ runner.temp }}/qa-prompt.txt" <, playwright-cli press , etc. - Run: playwright-cli screenshot --filename= (to capture failures) 2. Run the FULL QA test plan from the skill file 3. IMPORTANT: Take a screenshot after EVERY significant interaction to build a video. Save all screenshots sequentially to $QA_ARTIFACTS/frames/: mkdir -p $QA_ARTIFACTS/frames playwright-cli screenshot --filename=$QA_ARTIFACTS/frames/frame-001.png (increment the number for each screenshot: frame-002.png, frame-003.png, etc.) 4. Also save screenshots of failures or notable states into $QA_ARTIFACTS 5. Save report to $QA_ARTIFACTS as YYYY-MM-DD-NNN-${OS_LOWER}-report.md Do NOT create a new PR. Do NOT post PR comments. Do NOT commit or push anything. Skip tests not available in CI (file dialogs, GPU execution). PROMPT else cat > "${{ runner.temp }}/qa-prompt.txt" </dev/null || echo "Unknown") DIFF (truncated to 500 lines): $(head -500 "${{ runner.temp }}/pr-diff.txt" 2>/dev/null || echo "No diff available") Instructions: 1. Read the diff above to understand what changed in this PR 2. Use playwright-cli via Bash to navigate http://127.0.0.1:8188 - Run: playwright-cli goto http://127.0.0.1:8188 - Run: playwright-cli snapshot (after each navigation to get element refs) - Use: playwright-cli click , playwright-cli press , etc. - Run: playwright-cli screenshot --filename= (to save failures into $QA_ARTIFACTS) 3. IMPORTANT: Take a screenshot after EVERY significant interaction to build a video. Save all screenshots sequentially to $QA_ARTIFACTS/frames/: mkdir -p $QA_ARTIFACTS/frames playwright-cli screenshot --filename=$QA_ARTIFACTS/frames/frame-001.png (increment the number for each screenshot: frame-002.png, frame-003.png, etc.) 4. Test the specific UI areas affected by these changes 5. Also do a quick smoke test of core functionality (app loads, canvas renders, sidebar works) 6. Save a concise report to $QA_ARTIFACTS as YYYY-MM-DD-NNN-${OS_LOWER}-report.md Focus on: - Does the changed functionality work as expected? - Are there visual regressions in affected areas? - Do related features still work? Do NOT run the full QA test plan. Do NOT create a new PR. Do NOT post PR comments. Do NOT commit or push anything. Skip tests not available in CI (file dialogs, GPU execution). PROMPT fi - name: Run Claude QA shell: bash env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} CI: 'true' GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | MAX_TURNS=128 if [ "$QA_MODE" = "focused" ]; then MAX_TURNS=30; fi cat "${{ runner.temp }}/qa-prompt.txt" | claude --print \ --max-turns "$MAX_TURNS" \ --allowedTools "Bash(playwright-cli:*),Bash(date:*),Bash(ls:*),Bash(mkdir:*),Read,Write,Edit,Glob,Grep" - name: Collect artifacts if: always() shell: bash run: | mkdir -p "$QA_ARTIFACTS" ls -la "$QA_ARTIFACTS/" || true - name: Upload QA artifacts if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v6.2.0 with: name: qa-report-${{ runner.os }}-${{ github.run_id }} path: ${{ env.QA_ARTIFACTS }}/ retention-days: 14 report: needs: [resolve-matrix, qa] if: always() && (github.event.pull_request.number || github.event_name == 'push') runs-on: ubuntu-latest permissions: pull-requests: write steps: - name: Resolve PR number id: pr env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUM: ${{ github.event.pull_request.number }} run: | if [ -n "$PR_NUM" ]; then echo "number=$PR_NUM" >> "$GITHUB_OUTPUT" else # Push event: look up open PR for this branch NUM=$(gh pr list --repo "${{ github.repository }}" \ --head "${{ github.ref_name }}" --state open \ --json number --jq '.[0].number // empty') echo "number=${NUM}" >> "$GITHUB_OUTPUT" fi - name: Checkout repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Setup frontend uses: ./.github/actions/setup-frontend - name: Download QA artifacts uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: qa-artifacts pattern: qa-report-* - name: Install ffmpeg run: sudo apt-get update -qq && sudo apt-get install -y -qq ffmpeg >/dev/null 2>&1 - name: Stitch screenshots into video run: | for dir in qa-artifacts/qa-report-*/frames; do [ -d "$dir" ] || continue FRAME_COUNT=$(find "$dir" -name '*.png' | wc -l) if [ "$FRAME_COUNT" -eq 0 ]; then echo "No frames in $dir, skipping" continue fi PARENT=$(dirname "$dir") echo "Stitching $FRAME_COUNT frames from $dir" ffmpeg -y -framerate 2 -pattern_type glob -i "$dir/*.png" \ -c:v libx264 -preset ultrafast -crf 28 -pix_fmt yuv420p \ "$PARENT/qa-session.mp4" 2>/dev/null \ || echo "ffmpeg stitch failed for $dir (non-fatal)" done - name: Deploy videos to Cloudflare Pages id: deploy-videos env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} RAW_BRANCH: ${{ github.head_ref || github.ref_name }} run: | npm install -g wrangler@4.74.0 >/dev/null 2>&1 DEPLOY_DIR=$(mktemp -d) mkdir -p "$DEPLOY_DIR" for os in Linux macOS Windows; do VID="qa-artifacts/qa-report-${os}-${{ github.run_id }}/qa-session.mp4" if [ -f "$VID" ]; then cp "$VID" "$DEPLOY_DIR/qa-${os}.mp4" echo "Found ${os} video ($(du -h "$VID" | cut -f1))" # Generate GIF thumbnail: 8s starting at 10s, 480px wide, 8fps ffmpeg -y -ss 10 -i "$VID" -t 8 \ -vf "fps=8,scale=480:-1:flags=lanczos,split[s0][s1];[s0]palettegen=max_colors=64[p];[s1][p]paletteuse=dither=bayer" \ -loop 0 "$DEPLOY_DIR/qa-${os}-thumb.gif" 2>/dev/null \ || ffmpeg -y -i "$VID" -t 8 \ -vf "fps=8,scale=480:-1:flags=lanczos,split[s0][s1];[s0]palettegen=max_colors=64[p];[s1][p]paletteuse=dither=bayer" \ -loop 0 "$DEPLOY_DIR/qa-${os}-thumb.gif" 2>/dev/null \ || echo "GIF generation failed for ${os} (non-fatal)" fi done # Build video cards HTML CARDS="" ICONS_Linux="🐧" ICONS_macOS="🍎" ICONS_Windows="🪟" for os in Linux macOS Windows; do eval "ICON=\$ICONS_${os}" if [ -f "$DEPLOY_DIR/qa-${os}.mp4" ]; then CARDS="${CARDS}
${ICON} ${os}Download
" else CARDS="${CARDS}
No recording available
${ICON} ${os}Missing
" fi done cat > "$DEPLOY_DIR/index.html" <QA Session Recordings

QA Session Recordings

ComfyUI Frontend · Automated QA
${CARDS}
INDEXEOF # 404 page so Cloudflare Pages returns proper 404 for missing files # (instead of SPA fallback serving index.html) cat > "$DEPLOY_DIR/404.html" <<'ERROREOF' 404

404

File not found. The QA recording may have failed or been cancelled.

ERROREOF # Sanitize branch name for Cloudflare Pages URL (same rules CF uses) BRANCH=$(echo "$RAW_BRANCH" | sed 's/[^a-zA-Z0-9-]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//' | cut -c1-28) URL=$(wrangler pages deploy "$DEPLOY_DIR" \ --project-name="comfyui-qa-videos" \ --branch="$BRANCH" 2>&1 \ | grep -oE 'https://[a-zA-Z0-9.-]+\.pages\.dev\S*' | head -1) echo "url=${URL:-https://${BRANCH}.comfyui-qa-videos.pages.dev}" >> "$GITHUB_OUTPUT" echo "Deployed to: ${URL}" - name: Run video review env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | mkdir -p video-reviews for vid in qa-artifacts/qa-report-*/qa-session.mp4; do [ -f "$vid" ] || continue echo "::group::Reviewing $vid" pnpm exec tsx scripts/qa-video-review.ts \ --artifacts-dir qa-artifacts \ --output-dir video-reviews \ --video-file "$vid" \ --model gpt-4o || true echo "::endgroup::" done - name: Post unified QA comment on PR env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} VIDEO_BASE: ${{ steps.deploy-videos.outputs.url }} QA_MODE: ${{ needs.resolve-matrix.outputs.mode }} run: | RUN="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" COMMENT_MARKER="" MODE_BADGE="🔍 Focused" if [ "$QA_MODE" = "full" ]; then MODE_BADGE="🔬 Full (3-OS)"; fi # Build video section with GIF thumbnails linking to full videos VIDEO_SECTION="" for os in Linux macOS Windows; do GIF_URL="${VIDEO_BASE}/qa-${os}-thumb.gif" VID_URL="${VIDEO_BASE}/qa-${os}.mp4" if curl -sf --head "$VID_URL" >/dev/null 2>&1; then if curl -sf --head "$GIF_URL" >/dev/null 2>&1; then VIDEO_SECTION="${VIDEO_SECTION}[![${os} QA](${GIF_URL})](${VID_URL})"$'\n' else VIDEO_SECTION="${VIDEO_SECTION}[${os} video](${VID_URL})"$'\n' fi fi done # Build video review section from per-platform reports VIDEO_REVIEW="" for f in video-reviews/*-qa-video-report.md; do [ -f "$f" ] || continue [ -n "$VIDEO_REVIEW" ] && VIDEO_REVIEW="${VIDEO_REVIEW} --- " VIDEO_REVIEW="${VIDEO_REVIEW}$(cat "$f")" done VIDEO_REVIEW_SECTION="" if [ -n "$VIDEO_REVIEW" ]; then VIDEO_REVIEW_SECTION=$(cat < Video Review ${VIDEO_REVIEW} REVIEWEOF ) fi BODY=$(cat <- github.event.label.name == 'qa-changes' || github.event.label.name == 'qa-full' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} LABEL_NAME: ${{ github.event.label.name }} PR_NUMBER: ${{ steps.pr.outputs.number }} REPO: ${{ github.repository }} run: | [ -n "$PR_NUMBER" ] && gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$LABEL_NAME"