# Automated QA of ComfyUI frontend using Claude CLI + playwright-cli. # Two modes: # Focused (qa-changes label): Linux-only, tests areas affected by PR changes # Full (qa-full label): 3-OS matrix, full test plan name: 'PR: QA' on: # TODO: remove push trigger before merge push: branches: [sno-skills] pull_request: types: [labeled] branches: [main] workflow_dispatch: inputs: mode: description: 'QA mode' type: choice options: [focused, full] default: focused concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: resolve-matrix: runs-on: ubuntu-latest outputs: os: ${{ steps.set.outputs.os }} mode: ${{ steps.set.outputs.mode }} skip: ${{ steps.set.outputs.skip }} steps: - name: Determine QA mode id: set env: LABEL: ${{ github.event.label.name }} EVENT_ACTION: ${{ github.event.action }} EVENT_NAME: ${{ github.event_name }} INPUT_MODE: ${{ inputs.mode }} run: | FULL=false # Only run on label events if it's one of our labels if [ "$EVENT_ACTION" = "labeled" ] && \ [ "$LABEL" != "qa-changes" ] && [ "$LABEL" != "qa-full" ]; then echo "skip=true" >> "$GITHUB_OUTPUT" fi # Full QA triggers if [ "$EVENT_NAME" = "workflow_dispatch" ] && \ [ "$INPUT_MODE" = "full" ]; then FULL=true fi if [ "$LABEL" = "qa-full" ]; then FULL=true fi if [ "$FULL" = "true" ]; then echo 'os=["ubuntu-latest","macos-latest","windows-latest"]' >> "$GITHUB_OUTPUT" echo "mode=full" >> "$GITHUB_OUTPUT" else echo 'os=["ubuntu-latest"]' >> "$GITHUB_OUTPUT" echo "mode=focused" >> "$GITHUB_OUTPUT" fi echo "Mode: $([ "$FULL" = "true" ] && echo full || echo focused)" qa: needs: resolve-matrix if: needs.resolve-matrix.outputs.skip != 'true' strategy: fail-fast: false matrix: os: ${{ fromJson(needs.resolve-matrix.outputs.os) }} runs-on: ${{ matrix.os }} timeout-minutes: 60 permissions: pull-requests: write env: QA_MODE: ${{ needs.resolve-matrix.outputs.mode }} steps: - name: Set QA artifacts path shell: bash run: echo "QA_ARTIFACTS=$RUNNER_TEMP/qa-artifacts" >> "$GITHUB_ENV" - name: Checkout repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 ref: ${{ github.head_ref || github.ref }} token: ${{ secrets.GITHUB_TOKEN }} - name: Setup frontend uses: ./.github/actions/setup-frontend with: include_build_step: true - name: Setup and start ComfyUI server uses: ./.github/actions/setup-comfyui-server with: launch_server: 'true' - name: Wait for ComfyUI server shell: bash run: | echo "Waiting for ComfyUI server..." for i in $(seq 1 60); do if curl -sf http://127.0.0.1:8188/api/system_stats >/dev/null 2>&1; then echo "Server ready"; exit 0 fi; sleep 2 done echo "::error::Server timeout"; exit 1 - name: Install playwright-cli and Codex CLI shell: bash run: | npm install -g @playwright/cli@latest @openai/codex@latest # Verify playwright-cli is in PATH and install browser which playwright-cli playwright-cli --version || true npx playwright install chromium - name: Configure playwright-cli output shell: bash run: | mkdir -p "$QA_ARTIFACTS" .playwright # Auto-record video + save screenshots to artifacts dir cat > .playwright/cli.config.json < "${{ runner.temp }}/pr-diff.txt" 2>/dev/null || \ git diff origin/main...HEAD > "${{ runner.temp }}/pr-diff.txt" # Summarize changed files for the prompt echo "Changed files:" grep '^diff --git' "${{ runner.temp }}/pr-diff.txt" | \ sed 's|diff --git a/||;s| b/.*||' | sort -u | tee "${{ runner.temp }}/changed-files.txt" - name: Write QA prompt shell: bash env: BRANCH: ${{ github.head_ref || github.ref_name }} PR_NUM: ${{ github.event.pull_request.number || 'N/A' }} SHA: ${{ github.sha }} run: | OS_LOWER=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]') if [ "$QA_MODE" = "full" ]; then cat > "${{ runner.temp }}/qa-prompt.txt" < "${{ runner.temp }}/qa-prompt.txt" </dev/null || echo "Unknown") DIFF (truncated to 500 lines): $(head -500 "${{ runner.temp }}/pr-diff.txt" 2>/dev/null || echo "No diff available") CRITICAL: "playwright-cli" is already installed globally in PATH. Do NOT use pnpm dlx or npx. Chromium is already installed. Just run the commands directly. You MUST follow these exact steps in order: 1. playwright-cli open http://127.0.0.1:8188 2. playwright-cli video-start 3. playwright-cli snapshot 4. Test the changed UI areas (click, fill, navigate โ€” use snapshot between actions) 5. Quick smoke test: app loads, canvas renders, sidebar works 6. playwright-cli video-stop ${QA_ARTIFACTS}/qa-session.webm 7. Write report to ${QA_ARTIFACTS}/$(date +%Y-%m-%d)-001-${OS_LOWER}-report.md Do NOT skip steps 1-2 or 6-7. Do NOT use pnpm/npx to run playwright-cli. Do NOT create a PR, post PR comments, commit, or push anything. Skip tests not available in CI (file dialogs, GPU execution). PROMPT fi - name: Run Codex QA shell: bash env: CODEX_API_KEY: ${{ secrets.OPENAI_API_KEY }} CI: 'true' GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | codex exec \ --model gpt-5.4-mini \ --sandbox danger-full-access \ - < "${{ runner.temp }}/qa-prompt.txt" - name: Collect artifacts if: always() shell: bash run: | mkdir -p "$QA_ARTIFACTS" echo "=== QA artifacts ===" ls -la "$QA_ARTIFACTS/" 2>/dev/null | head -30 # Check for video from explicit video-stop command if [ -f "$QA_ARTIFACTS/qa-session.webm" ]; then echo "Found video: $QA_ARTIFACTS/qa-session.webm ($(du -h "$QA_ARTIFACTS/qa-session.webm" | cut -f1))" else echo "No qa-session.webm found at expected path" # Search for any .webm in artifacts dir or playwright-cli output VIDEO=$(find "$QA_ARTIFACTS" . -maxdepth 3 -name '*.webm' -not -path '*/node_modules/*' 2>/dev/null | head -1) if [ -n "$VIDEO" ]; then echo "Found fallback video: $VIDEO ($(du -h "$VIDEO" | cut -f1))" cp "$VIDEO" "$QA_ARTIFACTS/qa-session.webm" else echo "WARNING: No .webm video found anywhere" fi fi echo "=== Final artifacts ===" ls -la "$QA_ARTIFACTS/" 2>/dev/null | head -30 - name: Upload QA artifacts if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v6.2.0 with: name: qa-report-${{ runner.os }}-${{ github.run_id }} path: ${{ env.QA_ARTIFACTS }}/ retention-days: 14 report: needs: [resolve-matrix, qa] if: always() && (github.event.pull_request.number || github.event_name == 'push') runs-on: ubuntu-latest permissions: pull-requests: write steps: - name: Resolve PR number id: pr env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUM: ${{ github.event.pull_request.number }} run: | if [ -n "$PR_NUM" ]; then echo "number=$PR_NUM" >> "$GITHUB_OUTPUT" else # Push event: look up open PR for this branch NUM=$(gh pr list --repo "${{ github.repository }}" \ --head "${{ github.ref_name }}" --state open \ --json number --jq '.[0].number // empty') echo "number=${NUM}" >> "$GITHUB_OUTPUT" fi - name: Checkout repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Setup frontend uses: ./.github/actions/setup-frontend - name: Download QA artifacts uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: qa-artifacts pattern: qa-report-* - name: Normalize artifact layout run: | # download-artifact v7 extracts flat when there's only 1 artifact. # If qa-report-* subdirs don't exist, move flat files into the right subdir. if ! ls -d qa-artifacts/qa-report-* >/dev/null 2>&1; then echo "No qa-report-* subdirs found โ€” reorganizing flat download" # Detect OS from report filename (e.g. *-linux-report.md) for os in Linux macOS Windows; do OS_LOWER=$(echo "$os" | tr '[:upper:]' '[:lower:]') if ls qa-artifacts/*-${OS_LOWER}-report.md >/dev/null 2>&1; then DEST="qa-artifacts/qa-report-${os}-${{ github.run_id }}" mkdir -p "$DEST" find qa-artifacts -maxdepth 1 -type f -exec mv {} "$DEST/" \; [ -d qa-artifacts/videos ] && mv qa-artifacts/videos "$DEST/" echo "Moved flat files into $DEST" break fi done fi echo "=== Artifact structure ===" find qa-artifacts -type f 2>/dev/null | head -20 - name: Install ffmpeg run: sudo apt-get install -y -qq ffmpeg 2>&1 | tail -3 - name: Convert videos to mp4 run: | for dir in qa-artifacts/qa-report-*; do [ -d "$dir" ] || continue WEBM=$(find "$dir" -name '*.webm' -type f | head -1) if [ -z "$WEBM" ]; then echo "No .webm video in $dir, skipping" continue fi echo "Converting $WEBM ($(du -h "$WEBM" | cut -f1)) to mp4" ffmpeg -y -i "$WEBM" \ -c:v libx264 -preset ultrafast -crf 23 -pix_fmt yuv420p \ "$dir/qa-session.mp4" 2>&1 | tail -5 \ || echo "ffmpeg conversion failed for $WEBM (non-fatal)" if [ -f "$dir/qa-session.mp4" ]; then echo "Created: $dir/qa-session.mp4 ($(du -h "$dir/qa-session.mp4" | cut -f1))" fi done - name: Run video review env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | mkdir -p video-reviews for vid in qa-artifacts/qa-report-*/qa-session.mp4; do [ -f "$vid" ] || continue echo "::group::Reviewing $vid" pnpm exec tsx scripts/qa-video-review.ts \ --artifacts-dir qa-artifacts \ --output-dir video-reviews \ --video-file "$vid" \ --model gpt-4.1-mini || true echo "::endgroup::" done - name: Deploy to Cloudflare Pages id: deploy-videos env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} RAW_BRANCH: ${{ github.head_ref || github.ref_name }} run: | npm install -g wrangler@4.74.0 >/dev/null 2>&1 DEPLOY_DIR=$(mktemp -d) mkdir -p "$DEPLOY_DIR" for os in Linux macOS Windows; do VID="qa-artifacts/qa-report-${os}-${{ github.run_id }}/qa-session.mp4" if [ -f "$VID" ]; then cp "$VID" "$DEPLOY_DIR/qa-${os}.mp4" echo "Found ${os} video ($(du -h "$VID" | cut -f1))" # Generate GIF thumbnail ffmpeg -y -ss 10 -i "$VID" -t 8 \ -vf "fps=8,scale=480:-1:flags=lanczos,split[s0][s1];[s0]palettegen=max_colors=64[p];[s1][p]paletteuse=dither=bayer" \ -loop 0 "$DEPLOY_DIR/qa-${os}-thumb.gif" 2>/dev/null \ || ffmpeg -y -i "$VID" -t 8 \ -vf "fps=8,scale=480:-1:flags=lanczos,split[s0][s1];[s0]palettegen=max_colors=64[p];[s1][p]paletteuse=dither=bayer" \ -loop 0 "$DEPLOY_DIR/qa-${os}-thumb.gif" 2>/dev/null \ || echo "GIF generation failed for ${os} (non-fatal)" fi done # Build video cards and report sections CARDS="" ICONS_Linux="🐧" ICONS_macOS="🍎" ICONS_Windows="🪟" for os in Linux macOS Windows; do eval "ICON=\$ICONS_${os}" OS_LOWER=$(echo "$os" | tr '[:upper:]' '[:lower:]') # Copy GPT report if available REPORT_FILE="video-reviews/${OS_LOWER}-qa-video-report.md" REPORT_LINK="" REPORT_HTML="" if [ -f "$REPORT_FILE" ]; then cp "$REPORT_FILE" "$DEPLOY_DIR/report-${OS_LOWER}.md" REPORT_LINK="GPT Report" # Convert markdown to basic HTML for inline display REPORT_CONTENT=$(sed 's/&/\&/g; s//\>/g' "$REPORT_FILE" \ | sed 's/^## \(.*\)/

\1<\/h3>/; s/^# \(.*\)/

\1<\/h2>/' \ | sed 's/|\(.*\)|/\1<\/td><\/tr>/g' \ | sed '/^$/s/.*/
/') REPORT_HTML="
GPT Video Review
${REPORT_CONTENT}
" fi if [ -f "$DEPLOY_DIR/qa-${os}.mp4" ]; then CARDS="${CARDS}
${ICON} ${os}Download${REPORT_LINK}
${REPORT_HTML}
" else CARDS="${CARDS}
No recording available
${ICON} ${os}Missing
" fi done cat > "$DEPLOY_DIR/index.html" <QA Session Recordings

QA Session Recordings

ComfyUI Frontend · Automated QA
${CARDS}
INDEXEOF cat > "$DEPLOY_DIR/404.html" <<'ERROREOF' 404

404

File not found. The QA recording may have failed or been cancelled.

ERROREOF BRANCH=$(echo "$RAW_BRANCH" | sed 's/[^a-zA-Z0-9-]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//' | cut -c1-28) URL=$(wrangler pages deploy "$DEPLOY_DIR" \ --project-name="comfyui-qa-videos" \ --branch="$BRANCH" 2>&1 \ | grep -oE 'https://[a-zA-Z0-9.-]+\.pages\.dev\S*' | head -1) echo "url=${URL:-https://${BRANCH}.comfyui-qa-videos.pages.dev}" >> "$GITHUB_OUTPUT" echo "Deployed to: ${URL}" - name: Post unified QA comment on PR env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} VIDEO_BASE: ${{ steps.deploy-videos.outputs.url }} QA_MODE: ${{ needs.resolve-matrix.outputs.mode }} run: | RUN="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" COMMENT_MARKER="" MODE_BADGE="๐Ÿ” Focused" if [ "$QA_MODE" = "full" ]; then MODE_BADGE="๐Ÿ”ฌ Full (3-OS)"; fi # Build video section with GIF thumbnails linking to full videos VIDEO_SECTION="" for os in Linux macOS Windows; do GIF_URL="${VIDEO_BASE}/qa-${os}-thumb.gif" VID_URL="${VIDEO_BASE}/qa-${os}.mp4" if curl -sf --head "$VID_URL" >/dev/null 2>&1; then if curl -sf --head "$GIF_URL" >/dev/null 2>&1; then VIDEO_SECTION="${VIDEO_SECTION}[![${os} QA](${GIF_URL})](${VID_URL})"$'\n' else VIDEO_SECTION="${VIDEO_SECTION}[${os} video](${VID_URL})"$'\n' fi fi done # Build video review section from per-platform reports VIDEO_REVIEW="" for f in video-reviews/*-qa-video-report.md; do [ -f "$f" ] || continue [ -n "$VIDEO_REVIEW" ] && VIDEO_REVIEW="${VIDEO_REVIEW} --- " VIDEO_REVIEW="${VIDEO_REVIEW}$(cat "$f")" done VIDEO_REVIEW_SECTION="" if [ -n "$VIDEO_REVIEW" ]; then VIDEO_REVIEW_SECTION=$(cat < Video Review ${VIDEO_REVIEW} REVIEWEOF ) fi BODY=$(cat <- github.event.label.name == 'qa-changes' || github.event.label.name == 'qa-full' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} LABEL_NAME: ${{ github.event.label.name }} PR_NUMBER: ${{ steps.pr.outputs.number }} REPO: ${{ github.repository }} run: | [ -n "$PR_NUMBER" ] && gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$LABEL_NAME"