# Automated QA of ComfyUI frontend using Claude CLI + playwright-cli. # Two modes: # Focused (qa-changes label): Linux-only, tests areas affected by PR changes # Full (qa-full label): 3-OS matrix, full test plan name: 'PR: QA' on: # TODO: remove push trigger before merge push: branches: [sno-skills, sno-qa-*] pull_request: types: [labeled] branches: [main] workflow_dispatch: inputs: mode: description: 'QA mode' type: choice options: [focused, full] default: focused # TODO: restore concurrency group before merge (stuck-run issue on dev branches) # concurrency: # group: ${{ github.workflow }}-${{ github.ref }} # cancel-in-progress: true jobs: resolve-matrix: runs-on: ubuntu-latest outputs: os: ${{ steps.set.outputs.os }} mode: ${{ steps.set.outputs.mode }} skip: ${{ steps.set.outputs.skip }} steps: - name: Determine QA mode id: set env: LABEL: ${{ github.event.label.name }} EVENT_ACTION: ${{ github.event.action }} EVENT_NAME: ${{ github.event_name }} INPUT_MODE: ${{ inputs.mode }} run: | FULL=false # Only run on label events if it's one of our labels if [ "$EVENT_ACTION" = "labeled" ] && \ [ "$LABEL" != "qa-changes" ] && [ "$LABEL" != "qa-full" ]; then echo "skip=true" >> "$GITHUB_OUTPUT" fi # Full QA triggers if [ "$EVENT_NAME" = "workflow_dispatch" ] && \ [ "$INPUT_MODE" = "full" ]; then FULL=true fi if [ "$LABEL" = "qa-full" ]; then FULL=true fi if [ "$FULL" = "true" ]; then echo 'os=["ubuntu-latest","macos-latest","windows-latest"]' >> "$GITHUB_OUTPUT" echo "mode=full" >> "$GITHUB_OUTPUT" else echo 'os=["ubuntu-latest"]' >> "$GITHUB_OUTPUT" echo "mode=focused" >> "$GITHUB_OUTPUT" fi echo "Mode: $([ "$FULL" = "true" ] && echo full || echo focused)" qa: needs: resolve-matrix if: needs.resolve-matrix.outputs.skip != 'true' strategy: fail-fast: false matrix: os: ${{ fromJson(needs.resolve-matrix.outputs.os) }} runs-on: ${{ matrix.os }} timeout-minutes: 60 permissions: pull-requests: write env: QA_MODE: ${{ needs.resolve-matrix.outputs.mode }} steps: - name: Set QA artifacts path shell: bash run: echo "QA_ARTIFACTS=$RUNNER_TEMP/qa-artifacts" >> "$GITHUB_ENV" - name: Checkout repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 ref: ${{ github.head_ref || github.ref }} token: ${{ secrets.GITHUB_TOKEN }} - name: Setup frontend (PR branch) uses: ./.github/actions/setup-frontend with: include_build_step: true - name: Build main branch frontend for before comparison if: needs.resolve-matrix.outputs.mode == 'focused' shell: bash run: | # Save PR dist, build main dist mv dist dist-after git stash --include-untracked || true git checkout origin/main -- . pnpm install --frozen-lockfile # Use vite directly — skip typecheck since main may have TS errors # when built with the PR branch's lockfile/deps pnpm exec vite build mv dist dist-before # Restore PR branch files from HEAD (not 'git checkout -' which needs @{-1}) git checkout HEAD -- . git stash pop || true mv dist-after dist echo "Built both: dist-before/ (main) and dist/ (PR)" ls -la dist-before/index.html dist/index.html - name: Setup ComfyUI server (no launch) uses: ./.github/actions/setup-comfyui-server with: launch_server: 'false' - name: Install Playwright browser shell: bash run: | npx playwright install firefox mkdir -p "$QA_ARTIFACTS" - name: Get PR diff shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh pr diff ${{ github.event.pull_request.number || '' }} \ --repo ${{ github.repository }} > "${{ runner.temp }}/pr-diff.txt" 2>/dev/null || \ git diff origin/main...HEAD > "${{ runner.temp }}/pr-diff.txt" echo "Changed files:" grep '^diff --git' "${{ runner.temp }}/pr-diff.txt" | \ sed 's|diff --git a/||;s| b/.*||' | sort -u | tee "${{ runner.temp }}/changed-files.txt" # ── BEFORE run (main branch) ── - name: Start server with main branch frontend if: needs.resolve-matrix.outputs.mode == 'focused' shell: bash working-directory: ComfyUI run: | python main.py --cpu --multi-user --front-end-root ../dist-before & echo $! > /tmp/comfyui-server.pid for i in $(seq 1 60); do curl -sf http://127.0.0.1:8188/api/system_stats >/dev/null 2>&1 && echo "Server ready (main)" && exit 0 sleep 2 done echo "::error::Server timeout (main)"; exit 1 - name: Run BEFORE QA (main branch) if: needs.resolve-matrix.outputs.mode == 'focused' shell: bash env: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} run: | pnpm exec tsx scripts/qa-record.ts \ --mode before \ --diff "${{ runner.temp }}/pr-diff.txt" \ --output-dir "$QA_ARTIFACTS" \ --url http://127.0.0.1:8188 \ --test-plan .claude/skills/comfy-qa/SKILL.md - name: Stop server after BEFORE run if: needs.resolve-matrix.outputs.mode == 'focused' shell: bash run: | kill "$(cat /tmp/comfyui-server.pid)" 2>/dev/null || true sleep 2 # Ensure port is free for i in $(seq 1 10); do curl -sf http://127.0.0.1:8188/ >/dev/null 2>&1 || break sleep 1 done echo "Server stopped" # ── AFTER run (PR branch) ── - name: Start server with PR branch frontend shell: bash working-directory: ComfyUI run: | python main.py --cpu --multi-user --front-end-root ../dist & echo $! > /tmp/comfyui-server.pid for i in $(seq 1 60); do curl -sf http://127.0.0.1:8188/api/system_stats >/dev/null 2>&1 && echo "Server ready (PR)" && exit 0 sleep 2 done echo "::error::Server timeout (PR)"; exit 1 - name: Run AFTER QA (PR branch) shell: bash env: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} run: | pnpm exec tsx scripts/qa-record.ts \ --mode after \ --diff "${{ runner.temp }}/pr-diff.txt" \ --output-dir "$QA_ARTIFACTS" \ --url http://127.0.0.1:8188 \ --test-plan .claude/skills/comfy-qa/SKILL.md - name: Collect artifacts if: always() shell: bash run: | kill "$(cat /tmp/comfyui-server.pid)" 2>/dev/null || true mkdir -p "$QA_ARTIFACTS" echo "=== QA artifacts ===" ls -la "$QA_ARTIFACTS/" 2>/dev/null | head -30 # Check for after video if [ -f "$QA_ARTIFACTS/qa-session.webm" ]; then echo "Found after video: $(du -h "$QA_ARTIFACTS/qa-session.webm" | cut -f1)" else echo "No qa-session.webm found" VIDEO=$(find "$QA_ARTIFACTS" . -maxdepth 3 -name '*.webm' -not -path '*/node_modules/*' 2>/dev/null | head -1) if [ -n "$VIDEO" ]; then echo "Found fallback video: $VIDEO ($(du -h "$VIDEO" | cut -f1))" cp "$VIDEO" "$QA_ARTIFACTS/qa-session.webm" else echo "WARNING: No .webm video found anywhere" fi fi echo "=== Final artifacts ===" ls -la "$QA_ARTIFACTS/" 2>/dev/null | head -30 - name: Upload QA artifacts if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v6.2.0 with: name: qa-report-${{ runner.os }}-${{ github.run_id }} path: ${{ env.QA_ARTIFACTS }}/ retention-days: 14 report: needs: [resolve-matrix, qa] if: always() && (github.event.pull_request.number || github.event_name == 'push' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest permissions: pull-requests: write steps: - name: Resolve PR number id: pr env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUM: ${{ github.event.pull_request.number }} run: | if [ -n "$PR_NUM" ]; then echo "number=$PR_NUM" >> "$GITHUB_OUTPUT" else # Push event: look up open PR for this branch NUM=$(gh pr list --repo "${{ github.repository }}" \ --head "${{ github.ref_name }}" --state open \ --json number --jq '.[0].number // empty') echo "number=${NUM}" >> "$GITHUB_OUTPUT" fi - name: Checkout repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Setup frontend uses: ./.github/actions/setup-frontend - name: Download QA artifacts uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: qa-artifacts pattern: qa-report-* - name: Normalize artifact layout run: | # download-artifact v7 extracts flat when there's only 1 artifact. # If qa-report-* subdirs don't exist, move flat files into the right subdir. if ! ls -d qa-artifacts/qa-report-* >/dev/null 2>&1; then echo "No qa-report-* subdirs found — reorganizing flat download" # Detect OS from report filename (e.g. *-linux-report.md) for os in Linux macOS Windows; do OS_LOWER=$(echo "$os" | tr '[:upper:]' '[:lower:]') if ls qa-artifacts/*-${OS_LOWER}-report.md >/dev/null 2>&1; then DEST="qa-artifacts/qa-report-${os}-${{ github.run_id }}" mkdir -p "$DEST" find qa-artifacts -maxdepth 1 -type f -exec mv {} "$DEST/" \; [ -d qa-artifacts/videos ] && mv qa-artifacts/videos "$DEST/" echo "Moved flat files into $DEST" break fi done fi echo "=== Artifact structure ===" find qa-artifacts -type f 2>/dev/null | head -20 - name: Install ffmpeg run: | if command -v ffmpeg &>/dev/null; then echo "ffmpeg already installed" else echo "Downloading static ffmpeg..." TMP=$(mktemp -d) curl -sL "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz" | tar xJ -C "$TMP" sudo cp "$TMP"/ffmpeg-*/ffmpeg "$TMP"/ffmpeg-*/ffprobe /usr/local/bin/ rm -rf "$TMP" fi ffmpeg -version | head -1 ffprobe -version | head -1 - name: Convert videos to mp4 run: | convert_video() { local WEBM="$1" MP4="$2" echo "Converting $WEBM ($(du -h "$WEBM" | cut -f1)) to $MP4" ffmpeg -y -i "$WEBM" \ -c:v libx264 -preset ultrafast -crf 23 -pix_fmt yuv420p \ -movflags +faststart -g 60 \ "$MP4" 2>&1 | tail -5 \ || echo "ffmpeg conversion failed for $WEBM (non-fatal)" [ -f "$MP4" ] && echo "Created: $MP4 ($(du -h "$MP4" | cut -f1))" } for dir in qa-artifacts/qa-report-*; do [ -d "$dir" ] || continue # Convert after video (qa-session.webm) for name in qa-session qa-before-session; do if [ -f "$dir/${name}.webm" ] && [ -s "$dir/${name}.webm" ]; then convert_video "$dir/${name}.webm" "$dir/${name}.mp4" fi done # Fallback: find any non-empty webm not yet converted if [ ! -f "$dir/qa-session.mp4" ]; then WEBM=$(find "$dir" -name '*.webm' -type f -size +0c | head -1) [ -n "$WEBM" ] && convert_video "$WEBM" "$dir/qa-session.mp4" fi done - name: Build PR context for video review env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | PR_NUM="${{ github.event.pull_request.number || '' }}" if [ -n "$PR_NUM" ]; then { echo "### PR #${PR_NUM}" gh pr view "$PR_NUM" --repo "${{ github.repository }}" \ --json title,body --jq '"Title: \(.title)\n\nDescription:\n\(.body)"' 2>/dev/null || true echo "" echo "### Changed files" gh pr diff "$PR_NUM" --repo "${{ github.repository }}" 2>/dev/null \ | grep '^diff --git' | sed 's|diff --git a/||;s| b/.*||' | sort -u || true echo "" echo "### Diff (truncated to 300 lines)" gh pr diff "$PR_NUM" --repo "${{ github.repository }}" 2>/dev/null \ | head -300 || true } > pr-context.txt echo "PR context saved ($(wc -l < pr-context.txt) lines)" else echo "No PR number available, skipping PR context" fi - name: Run video review env: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} run: | mkdir -p video-reviews PR_CTX_FLAG="" if [ -f pr-context.txt ]; then PR_CTX_FLAG="--pr-context pr-context.txt" fi for vid in qa-artifacts/qa-report-*/qa-session.mp4; do [ -f "$vid" ] || continue DIR=$(dirname "$vid") BEFORE_FLAG="" if [ -f "$DIR/qa-before-session.mp4" ]; then BEFORE_FLAG="--before-video $DIR/qa-before-session.mp4" fi echo "::group::Reviewing $vid" pnpm exec tsx scripts/qa-video-review.ts \ --artifacts-dir qa-artifacts \ --output-dir video-reviews \ --video-file "$vid" \ --model gemini-2.5-flash $PR_CTX_FLAG $BEFORE_FLAG || true echo "::endgroup::" done - name: Generate regression test from QA report if: needs.resolve-matrix.outputs.mode == 'focused' env: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | PR_NUM="${{ steps.pr.outputs.number }}" PR_BRANCH="${{ github.head_ref || github.ref_name }}" if [ -z "$PR_NUM" ]; then echo "No PR number, skipping test generation" exit 0 fi # Find the first QA report REPORT=$(find video-reviews -name '*-qa-video-report.md' -type f | head -1) if [ ! -f "$REPORT" ]; then echo "No QA report found, skipping test generation" exit 0 fi # Ensure we have the PR diff DIFF_FILE="${{ runner.temp }}/pr-diff.txt" if [ ! -f "$DIFF_FILE" ]; then gh pr diff "$PR_NUM" --repo "${{ github.repository }}" > "$DIFF_FILE" 2>/dev/null || true fi # Generate the test TEST_NAME="qa-pr${PR_NUM}" TEST_PATH="browser_tests/tests/${TEST_NAME}.spec.ts" echo "::group::Generating regression test from QA report" pnpm exec tsx scripts/qa-generate-test.ts \ --qa-report "$REPORT" \ --pr-diff "$DIFF_FILE" \ --output "$TEST_PATH" || { echo "Test generation failed (non-fatal)" exit 0 } echo "::endgroup::" # Push to {branch}-add-qa-test TEST_BRANCH="${PR_BRANCH}-add-qa-test" git checkout -b "$TEST_BRANCH" HEAD 2>/dev/null || git checkout "$TEST_BRANCH" 2>/dev/null || true git add "$TEST_PATH" git commit -m "test: add QA regression test for PR #${PR_NUM}" || { echo "Nothing to commit" exit 0 } git push origin "$TEST_BRANCH" --force-with-lease || echo "Push failed (non-fatal)" echo "Pushed regression test to branch: $TEST_BRANCH" - name: Deploy to Cloudflare Pages id: deploy-videos env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} RAW_BRANCH: ${{ github.head_ref || github.ref_name }} run: | npm install -g wrangler@4.74.0 >/dev/null 2>&1 DEPLOY_DIR=$(mktemp -d) mkdir -p "$DEPLOY_DIR" for os in Linux macOS Windows; do DIR="qa-artifacts/qa-report-${os}-${{ github.run_id }}" for prefix in qa qa-before; do VID="${DIR}/${prefix}-session.mp4" if [ -f "$VID" ]; then DEST="$DEPLOY_DIR/${prefix}-${os}.mp4" cp "$VID" "$DEST" echo "Found ${prefix} ${os} video ($(du -h "$VID" | cut -f1))" fi done # Generate GIF thumbnail from after video if [ -f "$DEPLOY_DIR/qa-${os}.mp4" ]; then ffmpeg -y -ss 10 -i "$DEPLOY_DIR/qa-${os}.mp4" -t 8 \ -vf "fps=8,scale=480:-1:flags=lanczos,split[s0][s1];[s0]palettegen=max_colors=64[p];[s1][p]paletteuse=dither=bayer" \ -loop 0 "$DEPLOY_DIR/qa-${os}-thumb.gif" 2>/dev/null \ || echo "GIF generation failed for ${os} (non-fatal)" fi done # Build video cards and report sections CARDS="" ICONS_Linux="🐧" ICONS_macOS="🍎" ICONS_Windows="🪟" CARD_COUNT=0 DL_ICON="" for os in Linux macOS Windows; do eval "ICON=\$ICONS_${os}" OS_LOWER=$(echo "$os" | tr '[:upper:]' '[:lower:]') HAS_BEFORE=$([ -f "$DEPLOY_DIR/qa-before-${os}.mp4" ] && echo 1 || echo 0) HAS_AFTER=$([ -f "$DEPLOY_DIR/qa-${os}.mp4" ] && echo 1 || echo 0) [ "$HAS_AFTER" = "0" ] && continue REPORT_FILE="video-reviews/${OS_LOWER}-qa-video-report.md" REPORT_LINK="" REPORT_HTML="" if [ -f "$REPORT_FILE" ]; then cp "$REPORT_FILE" "$DEPLOY_DIR/report-${OS_LOWER}.md" REPORT_LINK="Report" REPORT_MD=$(cat "$REPORT_FILE" | sed 's/&/\&/g; s//\>/g') REPORT_HTML="
AI Comparative Review
${REPORT_MD}
" fi if [ "$HAS_BEFORE" = "1" ]; then # Side-by-side before/after layout CARDS="${CARDS}
${ICON}${os}${REPORT_LINK}
Before main
${DL_ICON}Before
After PR
${DL_ICON}After
${REPORT_HTML}
" else # Single video (full QA mode or no before available) CARDS="${CARDS}
${ICON}${os}${DL_ICON}Download${REPORT_LINK}
${REPORT_HTML}
" fi CARD_COUNT=$((CARD_COUNT + 1)) done cat > "$DEPLOY_DIR/index.html" <QA Session Recordings

QA Session Recordings

ComfyUI Frontend · Automated QA
${CARDS}
INDEXEOF cat > "$DEPLOY_DIR/404.html" <<'ERROREOF' 404

404

File not found. The QA recording may have failed or been cancelled.

ERROREOF BRANCH=$(echo "$RAW_BRANCH" | sed 's/[^a-zA-Z0-9-]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//' | cut -c1-28) URL=$(wrangler pages deploy "$DEPLOY_DIR" \ --project-name="comfy-qa" \ --branch="$BRANCH" 2>&1 \ | grep -oE 'https://[a-zA-Z0-9.-]+\.pages\.dev\S*' | head -1) echo "url=${URL:-https://${BRANCH}.comfy-qa.pages.dev}" >> "$GITHUB_OUTPUT" echo "Deployed to: ${URL}" - name: Post unified QA comment on PR env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} VIDEO_BASE: ${{ steps.deploy-videos.outputs.url }} QA_MODE: ${{ needs.resolve-matrix.outputs.mode }} run: | RUN="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" COMMENT_MARKER="" MODE_BADGE="🔍 Focused" if [ "$QA_MODE" = "full" ]; then MODE_BADGE="🔬 Full (3-OS)"; fi # Build video section with GIF thumbnails linking to full videos VIDEO_SECTION="" for os in Linux macOS Windows; do GIF_URL="${VIDEO_BASE}/qa-${os}-thumb.gif" VID_URL="${VIDEO_BASE}/qa-${os}.mp4" if curl -sf --head "$VID_URL" >/dev/null 2>&1; then if curl -sf --head "$GIF_URL" >/dev/null 2>&1; then VIDEO_SECTION="${VIDEO_SECTION}[![${os} QA](${GIF_URL})](${VID_URL})"$'\n' else VIDEO_SECTION="${VIDEO_SECTION}[${os} video](${VID_URL})"$'\n' fi fi done # Build video review section from per-platform reports VIDEO_REVIEW="" for f in video-reviews/*-qa-video-report.md; do [ -f "$f" ] || continue [ -n "$VIDEO_REVIEW" ] && VIDEO_REVIEW="${VIDEO_REVIEW} --- " VIDEO_REVIEW="${VIDEO_REVIEW}$(cat "$f")" done VIDEO_REVIEW_SECTION="" if [ -n "$VIDEO_REVIEW" ]; then VIDEO_REVIEW_SECTION=$(cat < Video Review ${VIDEO_REVIEW} REVIEWEOF ) fi BODY=$(cat <- github.event.label.name == 'qa-changes' || github.event.label.name == 'qa-full' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} LABEL_NAME: ${{ github.event.label.name }} PR_NUMBER: ${{ steps.pr.outputs.number }} REPO: ${{ github.repository }} run: | [ -n "$PR_NUMBER" ] && gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$LABEL_NAME"