mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-02-06 16:10:09 +00:00
feat: add test count display to Playwright PR comments
- Add extract-playwright-counts.mjs script to parse test results from Playwright reports - Update pr-playwright-deploy-and-comment.sh to extract and display test counts - Show overall summary with passed/failed/flaky/skipped counts - Display per-browser test counts inline with report links - Use dynamic status icons based on test results (✅/❌/⚠️) 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
snomiao
130
scripts/cicd/extract-playwright-counts.mjs
Executable file
130
scripts/cicd/extract-playwright-counts.mjs
Executable file
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
/**
|
||||
* Extract test counts from Playwright HTML report
|
||||
* @param {string} reportDir - Path to the playwright-report directory
|
||||
* @returns {Object} Test counts { passed, failed, flaky, skipped, total }
|
||||
*/
|
||||
function extractTestCounts(reportDir) {
|
||||
const counts = {
|
||||
passed: 0,
|
||||
failed: 0,
|
||||
flaky: 0,
|
||||
skipped: 0,
|
||||
total: 0
|
||||
};
|
||||
|
||||
try {
|
||||
// First, try to find report.json which Playwright generates with JSON reporter
|
||||
const jsonReportFile = path.join(reportDir, 'report.json');
|
||||
if (fs.existsSync(jsonReportFile)) {
|
||||
const reportJson = JSON.parse(fs.readFileSync(jsonReportFile, 'utf-8'));
|
||||
if (reportJson.stats) {
|
||||
counts.total = reportJson.stats.expected || 0;
|
||||
counts.passed = reportJson.stats.expected - (reportJson.stats.unexpected || 0) - (reportJson.stats.flaky || 0) - (reportJson.stats.skipped || 0);
|
||||
counts.failed = reportJson.stats.unexpected || 0;
|
||||
counts.flaky = reportJson.stats.flaky || 0;
|
||||
counts.skipped = reportJson.stats.skipped || 0;
|
||||
return counts;
|
||||
}
|
||||
}
|
||||
|
||||
// Try index.html - Playwright HTML report embeds data in a script tag
|
||||
const indexFile = path.join(reportDir, 'index.html');
|
||||
if (fs.existsSync(indexFile)) {
|
||||
const content = fs.readFileSync(indexFile, 'utf-8');
|
||||
|
||||
// Look for the embedded report data in various formats
|
||||
// Format 1: window.playwrightReportBase64
|
||||
let dataMatch = content.match(/window\.playwrightReportBase64\s*=\s*["']([^"']+)["']/);
|
||||
if (dataMatch) {
|
||||
try {
|
||||
const decodedData = Buffer.from(dataMatch[1], 'base64').toString('utf-8');
|
||||
const reportData = JSON.parse(decodedData);
|
||||
|
||||
if (reportData.stats) {
|
||||
counts.total = reportData.stats.expected || 0;
|
||||
counts.passed = reportData.stats.expected - (reportData.stats.unexpected || 0) - (reportData.stats.flaky || 0) - (reportData.stats.skipped || 0);
|
||||
counts.failed = reportData.stats.unexpected || 0;
|
||||
counts.flaky = reportData.stats.flaky || 0;
|
||||
counts.skipped = reportData.stats.skipped || 0;
|
||||
return counts;
|
||||
}
|
||||
} catch (e) {
|
||||
// Continue to try other formats
|
||||
}
|
||||
}
|
||||
|
||||
// Format 2: window.playwrightReport
|
||||
dataMatch = content.match(/window\.playwrightReport\s*=\s*({[\s\S]*?});/);
|
||||
if (dataMatch) {
|
||||
try {
|
||||
// Use Function constructor instead of eval for safety
|
||||
const reportData = (new Function('return ' + dataMatch[1]))();
|
||||
|
||||
if (reportData.stats) {
|
||||
counts.total = reportData.stats.expected || 0;
|
||||
counts.passed = reportData.stats.expected - (reportData.stats.unexpected || 0) - (reportData.stats.flaky || 0) - (reportData.stats.skipped || 0);
|
||||
counts.failed = reportData.stats.unexpected || 0;
|
||||
counts.flaky = reportData.stats.flaky || 0;
|
||||
counts.skipped = reportData.stats.skipped || 0;
|
||||
return counts;
|
||||
}
|
||||
} catch (e) {
|
||||
// Continue to try other formats
|
||||
}
|
||||
}
|
||||
|
||||
// Format 3: Look for stats in the HTML content directly
|
||||
// Playwright sometimes renders stats in the UI
|
||||
const statsMatch = content.match(/(\d+)\s+passed[^0-9]*(\d+)\s+failed[^0-9]*(\d+)\s+flaky[^0-9]*(\d+)\s+skipped/i);
|
||||
if (statsMatch) {
|
||||
counts.passed = parseInt(statsMatch[1]) || 0;
|
||||
counts.failed = parseInt(statsMatch[2]) || 0;
|
||||
counts.flaky = parseInt(statsMatch[3]) || 0;
|
||||
counts.skipped = parseInt(statsMatch[4]) || 0;
|
||||
counts.total = counts.passed + counts.failed + counts.flaky + counts.skipped;
|
||||
return counts;
|
||||
}
|
||||
|
||||
// Format 4: Try to extract from summary text patterns
|
||||
const passedMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+passed/i);
|
||||
const failedMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+failed/i);
|
||||
const flakyMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+flaky/i);
|
||||
const skippedMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+skipped/i);
|
||||
const totalMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+(?:total|ran)/i);
|
||||
|
||||
if (passedMatch) counts.passed = parseInt(passedMatch[1]) || 0;
|
||||
if (failedMatch) counts.failed = parseInt(failedMatch[1]) || 0;
|
||||
if (flakyMatch) counts.flaky = parseInt(flakyMatch[1]) || 0;
|
||||
if (skippedMatch) counts.skipped = parseInt(skippedMatch[1]) || 0;
|
||||
if (totalMatch) {
|
||||
counts.total = parseInt(totalMatch[1]) || 0;
|
||||
} else if (counts.passed || counts.failed || counts.flaky || counts.skipped) {
|
||||
counts.total = counts.passed + counts.failed + counts.flaky + counts.skipped;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error reading report from ${reportDir}:`, error);
|
||||
}
|
||||
|
||||
return counts;
|
||||
}
|
||||
|
||||
// Main execution
|
||||
const reportDir = process.argv[2];
|
||||
|
||||
if (!reportDir) {
|
||||
console.error('Usage: extract-playwright-counts.mjs <report-directory>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const counts = extractTestCounts(reportDir);
|
||||
|
||||
// Output as JSON for easy parsing in shell script
|
||||
console.log(JSON.stringify(counts));
|
||||
|
||||
export { extractTestCounts };
|
||||
@@ -159,12 +159,12 @@ else
|
||||
echo "Available reports:"
|
||||
ls -la reports/ 2>/dev/null || echo "Reports directory not found"
|
||||
|
||||
# Deploy all reports in parallel and collect URLs
|
||||
# Deploy all reports in parallel and collect URLs + test counts
|
||||
temp_dir=$(mktemp -d)
|
||||
pids=""
|
||||
i=0
|
||||
|
||||
# Start parallel deployments
|
||||
# Start parallel deployments and count extractions
|
||||
for browser in $BROWSERS; do
|
||||
if [ -d "reports/playwright-report-$browser" ]; then
|
||||
echo "Found report for $browser, deploying in parallel..."
|
||||
@@ -172,11 +172,20 @@ else
|
||||
url=$(deploy_report "reports/playwright-report-$browser" "$browser" "$cloudflare_branch")
|
||||
echo "$url" > "$temp_dir/$i.url"
|
||||
echo "Deployment result for $browser: $url"
|
||||
|
||||
# Extract test counts if Node.js is available
|
||||
if command -v node > /dev/null 2>&1 && [ -f "scripts/cicd/extract-playwright-counts.mjs" ]; then
|
||||
counts=$(node scripts/cicd/extract-playwright-counts.mjs "reports/playwright-report-$browser" 2>/dev/null || echo '{}')
|
||||
echo "$counts" > "$temp_dir/$i.counts"
|
||||
else
|
||||
echo '{}' > "$temp_dir/$i.counts"
|
||||
fi
|
||||
) &
|
||||
pids="$pids $!"
|
||||
else
|
||||
echo "Report not found for $browser at reports/playwright-report-$browser"
|
||||
echo "failed" > "$temp_dir/$i.url"
|
||||
echo '{}' > "$temp_dir/$i.counts"
|
||||
fi
|
||||
i=$((i + 1))
|
||||
done
|
||||
@@ -186,8 +195,9 @@ else
|
||||
wait $pid
|
||||
done
|
||||
|
||||
# Collect URLs in order
|
||||
# Collect URLs and counts in order
|
||||
urls=""
|
||||
all_counts=""
|
||||
i=0
|
||||
for browser in $BROWSERS; do
|
||||
if [ -f "$temp_dir/$i.url" ]; then
|
||||
@@ -200,37 +210,143 @@ else
|
||||
else
|
||||
urls="$urls $url"
|
||||
fi
|
||||
|
||||
if [ -f "$temp_dir/$i.counts" ]; then
|
||||
counts=$(cat "$temp_dir/$i.counts")
|
||||
else
|
||||
counts="{}"
|
||||
fi
|
||||
if [ -z "$all_counts" ]; then
|
||||
all_counts="$counts"
|
||||
else
|
||||
all_counts="$all_counts|$counts"
|
||||
fi
|
||||
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
# Clean up temp directory
|
||||
rm -rf "$temp_dir"
|
||||
|
||||
# Calculate total test counts across all browsers
|
||||
total_passed=0
|
||||
total_failed=0
|
||||
total_flaky=0
|
||||
total_skipped=0
|
||||
total_tests=0
|
||||
|
||||
# Parse counts and calculate totals
|
||||
IFS='|'
|
||||
set -- $all_counts
|
||||
for counts_json; do
|
||||
if [ "$counts_json" != "{}" ] && [ -n "$counts_json" ]; then
|
||||
# Parse JSON counts using simple grep/sed if jq is not available
|
||||
if command -v jq > /dev/null 2>&1; then
|
||||
passed=$(echo "$counts_json" | jq -r '.passed // 0')
|
||||
failed=$(echo "$counts_json" | jq -r '.failed // 0')
|
||||
flaky=$(echo "$counts_json" | jq -r '.flaky // 0')
|
||||
skipped=$(echo "$counts_json" | jq -r '.skipped // 0')
|
||||
total=$(echo "$counts_json" | jq -r '.total // 0')
|
||||
else
|
||||
# Fallback parsing without jq
|
||||
passed=$(echo "$counts_json" | sed -n 's/.*"passed":\([0-9]*\).*/\1/p')
|
||||
failed=$(echo "$counts_json" | sed -n 's/.*"failed":\([0-9]*\).*/\1/p')
|
||||
flaky=$(echo "$counts_json" | sed -n 's/.*"flaky":\([0-9]*\).*/\1/p')
|
||||
skipped=$(echo "$counts_json" | sed -n 's/.*"skipped":\([0-9]*\).*/\1/p')
|
||||
total=$(echo "$counts_json" | sed -n 's/.*"total":\([0-9]*\).*/\1/p')
|
||||
fi
|
||||
|
||||
total_passed=$((total_passed + ${passed:-0}))
|
||||
total_failed=$((total_failed + ${failed:-0}))
|
||||
total_flaky=$((total_flaky + ${flaky:-0}))
|
||||
total_skipped=$((total_skipped + ${skipped:-0}))
|
||||
total_tests=$((total_tests + ${total:-0}))
|
||||
fi
|
||||
done
|
||||
unset IFS
|
||||
|
||||
# Determine overall status
|
||||
if [ $total_failed -gt 0 ]; then
|
||||
status_icon="❌"
|
||||
status_text="Some tests failed"
|
||||
elif [ $total_flaky -gt 0 ]; then
|
||||
status_icon="⚠️"
|
||||
status_text="Tests passed with flaky tests"
|
||||
elif [ $total_tests -gt 0 ]; then
|
||||
status_icon="✅"
|
||||
status_text="All tests passed!"
|
||||
else
|
||||
status_icon="⚠️"
|
||||
status_text="No test results found"
|
||||
fi
|
||||
|
||||
# Generate completion comment
|
||||
comment="$COMMENT_MARKER
|
||||
## 🎭 Playwright Test Results
|
||||
|
||||
✅ **Tests completed successfully!**
|
||||
$status_icon **$status_text**
|
||||
|
||||
⏰ Completed at: $(date -u '+%m/%d/%Y, %I:%M:%S %p') UTC
|
||||
⏰ Completed at: $(date -u '+%m/%d/%Y, %I:%M:%S %p') UTC"
|
||||
|
||||
# Add summary counts if we have test data
|
||||
if [ $total_tests -gt 0 ]; then
|
||||
comment="$comment
|
||||
|
||||
### 📈 Summary
|
||||
- **Total Tests:** $total_tests
|
||||
- **Passed:** $total_passed ✅
|
||||
- **Failed:** $total_failed $([ $total_failed -gt 0 ] && echo '❌' || echo '')
|
||||
- **Flaky:** $total_flaky $([ $total_flaky -gt 0 ] && echo '⚠️' || echo '')
|
||||
- **Skipped:** $total_skipped $([ $total_skipped -gt 0 ] && echo '⏭️' || echo '')"
|
||||
fi
|
||||
|
||||
comment="$comment
|
||||
|
||||
### 📊 Test Reports by Browser"
|
||||
|
||||
# Add browser results
|
||||
# Add browser results with individual counts
|
||||
i=0
|
||||
for browser in $BROWSERS; do
|
||||
IFS='|'
|
||||
set -- $all_counts
|
||||
for counts_json; do
|
||||
# Get browser name
|
||||
browser=$(echo "$BROWSERS" | cut -d' ' -f$((i + 1)))
|
||||
# Get URL at position i
|
||||
url=$(echo "$urls" | cut -d' ' -f$((i + 1)))
|
||||
|
||||
if [ "$url" != "failed" ] && [ -n "$url" ]; then
|
||||
# Parse individual browser counts
|
||||
if [ "$counts_json" != "{}" ] && [ -n "$counts_json" ]; then
|
||||
if command -v jq > /dev/null 2>&1; then
|
||||
b_passed=$(echo "$counts_json" | jq -r '.passed // 0')
|
||||
b_failed=$(echo "$counts_json" | jq -r '.failed // 0')
|
||||
b_flaky=$(echo "$counts_json" | jq -r '.flaky // 0')
|
||||
b_total=$(echo "$counts_json" | jq -r '.total // 0')
|
||||
else
|
||||
b_passed=$(echo "$counts_json" | sed -n 's/.*"passed":\([0-9]*\).*/\1/p')
|
||||
b_failed=$(echo "$counts_json" | sed -n 's/.*"failed":\([0-9]*\).*/\1/p')
|
||||
b_flaky=$(echo "$counts_json" | sed -n 's/.*"flaky":\([0-9]*\).*/\1/p')
|
||||
b_total=$(echo "$counts_json" | sed -n 's/.*"total":\([0-9]*\).*/\1/p')
|
||||
fi
|
||||
|
||||
if [ -n "$b_total" ] && [ "$b_total" != "0" ]; then
|
||||
counts_str=" (✅ $b_passed / ❌ $b_failed / ⚠️ $b_flaky)"
|
||||
else
|
||||
counts_str=""
|
||||
fi
|
||||
else
|
||||
counts_str=""
|
||||
fi
|
||||
|
||||
comment="$comment
|
||||
- ✅ **${browser}**: [View Report](${url})"
|
||||
- ✅ **${browser}**${counts_str}: [View Report](${url})"
|
||||
else
|
||||
comment="$comment
|
||||
- ❌ **${browser}**: Deployment failed"
|
||||
fi
|
||||
i=$((i + 1))
|
||||
done
|
||||
unset IFS
|
||||
|
||||
comment="$comment
|
||||
|
||||
|
||||
Reference in New Issue
Block a user