From 5ae6efdab3a0a751a40eb8fe60b4a846118b2990 Mon Sep 17 00:00:00 2001
From: snomiao <snomiao@gmail.com>
Date: Tue, 9 Sep 2025 22:55:21 +0000
Subject: [PATCH] feat: add test count display to Playwright PR comments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add extract-playwright-counts.mjs script to parse test results from Playwright reports
- Update pr-playwright-deploy-and-comment.sh to extract and display test counts
- Show overall summary with passed/failed/flaky/skipped counts
- Display per-browser test counts inline with report links
- Use dynamic status icons based on test results (✅/❌/⚠️)

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 scripts/cicd/extract-playwright-counts.mjs    | 130 +++++++++++++++++
 .../cicd/pr-playwright-deploy-and-comment.sh  | 132 ++++++++++++++++--
 2 files changed, 254 insertions(+), 8 deletions(-)
 create mode 100755 scripts/cicd/extract-playwright-counts.mjs

diff --git a/scripts/cicd/extract-playwright-counts.mjs b/scripts/cicd/extract-playwright-counts.mjs
new file mode 100755
index 000000000..cc34b2ce1
--- /dev/null
+++ b/scripts/cicd/extract-playwright-counts.mjs
@@ -0,0 +1,130 @@
+#!/usr/bin/env node
+
+import fs from 'fs';
+import path from 'path';
+
+/**
+ * Extract test counts from Playwright HTML report
+ * @param {string} reportDir - Path to the playwright-report directory
+ * @returns {Object} Test counts { passed, failed, flaky, skipped, total }
+ */
+function extractTestCounts(reportDir) {
+  const counts = {
+    passed: 0,
+    failed: 0,
+    flaky: 0,
+    skipped: 0,
+    total: 0
+  };
+
+  try {
+    // First, try to find report.json which Playwright generates with JSON reporter
+    const jsonReportFile = path.join(reportDir, 'report.json');
+    if (fs.existsSync(jsonReportFile)) {
+      const reportJson = JSON.parse(fs.readFileSync(jsonReportFile, 'utf-8'));
+      if (reportJson.stats) {
+        counts.total = reportJson.stats.expected || 0;
+        counts.passed = reportJson.stats.expected - (reportJson.stats.unexpected || 0) - (reportJson.stats.flaky || 0) - (reportJson.stats.skipped || 0);
+        counts.failed = reportJson.stats.unexpected || 0;
+        counts.flaky = reportJson.stats.flaky || 0;
+        counts.skipped = reportJson.stats.skipped || 0;
+        return counts;
+      }
+    }
+
+    // Try index.html - Playwright HTML report embeds data in a script tag
+    const indexFile = path.join(reportDir, 'index.html');
+    if (fs.existsSync(indexFile)) {
+      const content = fs.readFileSync(indexFile, 'utf-8');
+      
+      // Look for the embedded report data in various formats
+      // Format 1: window.playwrightReportBase64
+      let dataMatch = content.match(/window\.playwrightReportBase64\s*=\s*["']([^"']+)["']/);
+      if (dataMatch) {
+        try {
+          const decodedData = Buffer.from(dataMatch[1], 'base64').toString('utf-8');
+          const reportData = JSON.parse(decodedData);
+          
+          if (reportData.stats) {
+            counts.total = reportData.stats.expected || 0;
+            counts.passed = reportData.stats.expected - (reportData.stats.unexpected || 0) - (reportData.stats.flaky || 0) - (reportData.stats.skipped || 0);
+            counts.failed = reportData.stats.unexpected || 0;
+            counts.flaky = reportData.stats.flaky || 0;
+            counts.skipped = reportData.stats.skipped || 0;
+            return counts;
+          }
+        } catch (e) {
+          // Continue to try other formats
+        }
+      }
+      
+      // Format 2: window.playwrightReport
+      dataMatch = content.match(/window\.playwrightReport\s*=\s*({[\s\S]*?});/);
+      if (dataMatch) {
+        try {
+          // Use Function constructor instead of eval for safety
+          const reportData = (new Function('return ' + dataMatch[1]))();
+          
+          if (reportData.stats) {
+            counts.total = reportData.stats.expected || 0;
+            counts.passed = reportData.stats.expected - (reportData.stats.unexpected || 0) - (reportData.stats.flaky || 0) - (reportData.stats.skipped || 0);
+            counts.failed = reportData.stats.unexpected || 0;
+            counts.flaky = reportData.stats.flaky || 0;
+            counts.skipped = reportData.stats.skipped || 0;
+            return counts;
+          }
+        } catch (e) {
+          // Continue to try other formats
+        }
+      }
+
+      // Format 3: Look for stats in the HTML content directly
+      // Playwright sometimes renders stats in the UI
+      const statsMatch = content.match(/(\d+)\s+passed[^0-9]*(\d+)\s+failed[^0-9]*(\d+)\s+flaky[^0-9]*(\d+)\s+skipped/i);
+      if (statsMatch) {
+        counts.passed = parseInt(statsMatch[1]) || 0;
+        counts.failed = parseInt(statsMatch[2]) || 0;
+        counts.flaky = parseInt(statsMatch[3]) || 0;
+        counts.skipped = parseInt(statsMatch[4]) || 0;
+        counts.total = counts.passed + counts.failed + counts.flaky + counts.skipped;
+        return counts;
+      }
+
+      // Format 4: Try to extract from summary text patterns
+      const passedMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+passed/i);
+      const failedMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+failed/i);
+      const flakyMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+flaky/i);
+      const skippedMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+skipped/i);
+      const totalMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+(?:total|ran)/i);
+      
+      if (passedMatch) counts.passed = parseInt(passedMatch[1]) || 0;
+      if (failedMatch) counts.failed = parseInt(failedMatch[1]) || 0;
+      if (flakyMatch) counts.flaky = parseInt(flakyMatch[1]) || 0;
+      if (skippedMatch) counts.skipped = parseInt(skippedMatch[1]) || 0;
+      if (totalMatch) {
+        counts.total = parseInt(totalMatch[1]) || 0;
+      } else if (counts.passed || counts.failed || counts.flaky || counts.skipped) {
+        counts.total = counts.passed + counts.failed + counts.flaky + counts.skipped;
+      }
+    }
+  } catch (error) {
+    console.error(`Error reading report from ${reportDir}:`, error);
+  }
+
+  return counts;
+}
+
+// Main execution
+const reportDir = process.argv[2];
+
+if (!reportDir) {
+  console.error('Usage: extract-playwright-counts.mjs <report-directory>');
+  process.exit(1);
+}
+
+const counts = extractTestCounts(reportDir);
+
+// Output as JSON for easy parsing in shell script
+console.log(JSON.stringify(counts));
+
+export { extractTestCounts };
\ No newline at end of file
diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.sh b/scripts/cicd/pr-playwright-deploy-and-comment.sh
index 767a7f514..13c7ed3fe 100755
--- a/scripts/cicd/pr-playwright-deploy-and-comment.sh
+++ b/scripts/cicd/pr-playwright-deploy-and-comment.sh
@@ -159,12 +159,12 @@ else
     echo "Available reports:"
     ls -la reports/ 2>/dev/null || echo "Reports directory not found"
     
-    # Deploy all reports in parallel and collect URLs
+    # Deploy all reports in parallel and collect URLs + test counts
     temp_dir=$(mktemp -d)
     pids=""
     i=0
     
-    # Start parallel deployments
+    # Start parallel deployments and count extractions
     for browser in $BROWSERS; do
         if [ -d "reports/playwright-report-$browser" ]; then
             echo "Found report for $browser, deploying in parallel..."
@@ -172,11 +172,20 @@ else
                 url=$(deploy_report "reports/playwright-report-$browser" "$browser" "$cloudflare_branch")
                 echo "$url" > "$temp_dir/$i.url"
                 echo "Deployment result for $browser: $url"
+                
+                # Extract test counts if Node.js is available
+                if command -v node > /dev/null 2>&1 && [ -f "scripts/cicd/extract-playwright-counts.mjs" ]; then
+                    counts=$(node scripts/cicd/extract-playwright-counts.mjs "reports/playwright-report-$browser" 2>/dev/null || echo '{}')
+                    echo "$counts" > "$temp_dir/$i.counts"
+                else
+                    echo '{}' > "$temp_dir/$i.counts"
+                fi
             ) &
             pids="$pids $!"
         else
             echo "Report not found for $browser at reports/playwright-report-$browser"
             echo "failed" > "$temp_dir/$i.url"
+            echo '{}' > "$temp_dir/$i.counts"
         fi
         i=$((i + 1))
     done
@@ -186,8 +195,9 @@ else
         wait $pid
     done
     
-    # Collect URLs in order
+    # Collect URLs and counts in order
     urls=""
+    all_counts=""
     i=0
     for browser in $BROWSERS; do
         if [ -f "$temp_dir/$i.url" ]; then
@@ -200,37 +210,143 @@ else
         else
             urls="$urls $url"
         fi
+        
+        if [ -f "$temp_dir/$i.counts" ]; then
+            counts=$(cat "$temp_dir/$i.counts")
+        else
+            counts="{}"
+        fi
+        if [ -z "$all_counts" ]; then
+            all_counts="$counts"
+        else
+            all_counts="$all_counts|$counts"
+        fi
+        
         i=$((i + 1))
     done
     
     # Clean up temp directory
     rm -rf "$temp_dir"
     
+    # Calculate total test counts across all browsers
+    total_passed=0
+    total_failed=0
+    total_flaky=0
+    total_skipped=0
+    total_tests=0
+    
+    # Parse counts and calculate totals
+    IFS='|'
+    set -- $all_counts
+    for counts_json; do
+        if [ "$counts_json" != "{}" ] && [ -n "$counts_json" ]; then
+            # Parse JSON counts using simple grep/sed if jq is not available
+            if command -v jq > /dev/null 2>&1; then
+                passed=$(echo "$counts_json" | jq -r '.passed // 0')
+                failed=$(echo "$counts_json" | jq -r '.failed // 0')
+                flaky=$(echo "$counts_json" | jq -r '.flaky // 0')
+                skipped=$(echo "$counts_json" | jq -r '.skipped // 0')
+                total=$(echo "$counts_json" | jq -r '.total // 0')
+            else
+                # Fallback parsing without jq
+                passed=$(echo "$counts_json" | sed -n 's/.*"passed":\([0-9]*\).*/\1/p')
+                failed=$(echo "$counts_json" | sed -n 's/.*"failed":\([0-9]*\).*/\1/p')
+                flaky=$(echo "$counts_json" | sed -n 's/.*"flaky":\([0-9]*\).*/\1/p')
+                skipped=$(echo "$counts_json" | sed -n 's/.*"skipped":\([0-9]*\).*/\1/p')
+                total=$(echo "$counts_json" | sed -n 's/.*"total":\([0-9]*\).*/\1/p')
+            fi
+            
+            total_passed=$((total_passed + ${passed:-0}))
+            total_failed=$((total_failed + ${failed:-0}))
+            total_flaky=$((total_flaky + ${flaky:-0}))
+            total_skipped=$((total_skipped + ${skipped:-0}))
+            total_tests=$((total_tests + ${total:-0}))
+        fi
+    done
+    unset IFS
+    
+    # Determine overall status
+    if [ $total_failed -gt 0 ]; then
+        status_icon="❌"
+        status_text="Some tests failed"
+    elif [ $total_flaky -gt 0 ]; then
+        status_icon="⚠️"
+        status_text="Tests passed with flaky tests"
+    elif [ $total_tests -gt 0 ]; then
+        status_icon="✅"
+        status_text="All tests passed!"
+    else
+        status_icon="⚠️"
+        status_text="No test results found"
+    fi
+    
     # Generate completion comment
     comment="$COMMENT_MARKER
 ## 🎭 Playwright Test Results
 
-✅ **Tests completed successfully!**
+$status_icon **$status_text**
 
-⏰ Completed at: $(date -u '+%m/%d/%Y, %I:%M:%S %p') UTC
+⏰ Completed at: $(date -u '+%m/%d/%Y, %I:%M:%S %p') UTC"
+
+    # Add summary counts if we have test data
+    if [ $total_tests -gt 0 ]; then
+        comment="$comment
+
+### 📈 Summary
+- **Total Tests:** $total_tests
+- **Passed:** $total_passed ✅
+- **Failed:** $total_failed $([ $total_failed -gt 0 ] && echo '❌' || echo '')
+- **Flaky:** $total_flaky $([ $total_flaky -gt 0 ] && echo '⚠️' || echo '')
+- **Skipped:** $total_skipped $([ $total_skipped -gt 0 ] && echo '⏭️' || echo '')"
+    fi
+    
+    comment="$comment
 
 ### 📊 Test Reports by Browser"
     
-    # Add browser results
+    # Add browser results with individual counts
     i=0
-    for browser in $BROWSERS; do
+    IFS='|'
+    set -- $all_counts
+    for counts_json; do
+        # Get browser name
+        browser=$(echo "$BROWSERS" | cut -d' ' -f$((i + 1)))
         # Get URL at position i
         url=$(echo "$urls" | cut -d' ' -f$((i + 1)))
         
         if [ "$url" != "failed" ] && [ -n "$url" ]; then
+            # Parse individual browser counts
+            if [ "$counts_json" != "{}" ] && [ -n "$counts_json" ]; then
+                if command -v jq > /dev/null 2>&1; then
+                    b_passed=$(echo "$counts_json" | jq -r '.passed // 0')
+                    b_failed=$(echo "$counts_json" | jq -r '.failed // 0')
+                    b_flaky=$(echo "$counts_json" | jq -r '.flaky // 0')
+                    b_total=$(echo "$counts_json" | jq -r '.total // 0')
+                else
+                    b_passed=$(echo "$counts_json" | sed -n 's/.*"passed":\([0-9]*\).*/\1/p')
+                    b_failed=$(echo "$counts_json" | sed -n 's/.*"failed":\([0-9]*\).*/\1/p')
+                    b_flaky=$(echo "$counts_json" | sed -n 's/.*"flaky":\([0-9]*\).*/\1/p')
+                    b_total=$(echo "$counts_json" | sed -n 's/.*"total":\([0-9]*\).*/\1/p')
+                fi
+                
+                if [ -n "$b_total" ] && [ "$b_total" != "0" ]; then
+                    counts_str=" (✅ $b_passed / ❌ $b_failed / ⚠️ $b_flaky)"
+                else
+                    counts_str=""
+                fi
+            else
+                counts_str=""
+            fi
+            
             comment="$comment
-- ✅ **${browser}**: [View Report](${url})"
+- ✅ **${browser}**${counts_str}: [View Report](${url})"
         else
             comment="$comment
 - ❌ **${browser}**: Deployment failed"
         fi
         i=$((i + 1))
     done
+    unset IFS
     
     comment="$comment