diff --git a/scripts/cicd/extract-playwright-counts.ts b/scripts/cicd/extract-playwright-counts.ts index 89926ab98..027206673 100755 --- a/scripts/cicd/extract-playwright-counts.ts +++ b/scripts/cicd/extract-playwright-counts.ts @@ -52,6 +52,14 @@ interface FailingTest { line: number error: string tracePath?: string + failureType?: 'screenshot' | 'expectation' | 'timeout' | 'other' +} + +interface FailureTypeCounts { + screenshot: number + expectation: number + timeout: number + other: number } interface TestCounts { @@ -61,6 +69,48 @@ interface TestCounts { skipped: number total: number failingTests?: FailingTest[] + failureTypes?: FailureTypeCounts +} + +/** + * Categorize the failure type based on error message + */ +function categorizeFailureType( + error: string, + status: string +): 'screenshot' | 'expectation' | 'timeout' | 'other' { + if (status === 'timedOut') { + return 'timeout' + } + + const errorLower = error.toLowerCase() + + // Screenshot-related errors + if ( + errorLower.includes('screenshot') || + errorLower.includes('snapshot') || + errorLower.includes('toHaveScreenshot') || + errorLower.includes('image comparison') || + errorLower.includes('pixel') || + errorLower.includes('visual') + ) { + return 'screenshot' + } + + // Expectation errors + if ( + errorLower.includes('expect') || + errorLower.includes('assertion') || + errorLower.includes('toEqual') || + errorLower.includes('toBe') || + errorLower.includes('toContain') || + errorLower.includes('toHave') || + errorLower.includes('toMatch') + ) { + return 'expectation' + } + + return 'other' } /** @@ -94,12 +144,15 @@ function extractFailingTests( } } + const failureType = categorizeFailureType(error, result.status) + failingTests.push({ name: test.title, filePath: test.location?.file || 'unknown', line: test.location?.line || 0, error: error.split('\n')[0], // First line of error - tracePath + tracePath, + failureType }) } } @@ -126,7 +179,13 @@ function extractTestCounts(reportDir: string): TestCounts { flaky: 0, skipped: 0, total: 0, - failingTests: [] + failingTests: [], + failureTypes: { + screenshot: 0, + expectation: 0, + timeout: 0, + other: 0 + } } try { @@ -155,6 +214,14 @@ function extractTestCounts(reportDir: string): TestCounts { } } + // Count failure types + if (counts.failingTests) { + for (const test of counts.failingTests) { + const type = test.failureType || 'other' + counts.failureTypes![type]++ + } + } + return counts } } @@ -195,6 +262,14 @@ function extractTestCounts(reportDir: string): TestCounts { } } + // Count failure types + if (counts.failingTests) { + for (const test of counts.failingTests) { + const type = test.failureType || 'other' + counts.failureTypes![type]++ + } + } + return counts } } catch (e) { @@ -230,6 +305,14 @@ function extractTestCounts(reportDir: string): TestCounts { } } + // Count failure types + if (counts.failingTests) { + for (const test of counts.failingTests) { + const type = test.failureType || 'other' + counts.failureTypes![type]++ + } + } + return counts } } catch (e) { diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.sh b/scripts/cicd/pr-playwright-deploy-and-comment.sh index 9940b4447..2a060f1fd 100755 --- a/scripts/cicd/pr-playwright-deploy-and-comment.sh +++ b/scripts/cicd/pr-playwright-deploy-and-comment.sh @@ -252,6 +252,10 @@ else total_flaky=0 total_skipped=0 total_tests=0 + total_screenshot_failures=0 + total_expectation_failures=0 + total_timeout_failures=0 + total_other_failures=0 # Parse counts and calculate totals IFS='|' read -r -a counts_array <<< "$all_counts" @@ -265,6 +269,10 @@ else flaky=$(echo "$counts_json" | jq -r '.flaky // 0') skipped=$(echo "$counts_json" | jq -r '.skipped // 0') total=$(echo "$counts_json" | jq -r '.total // 0') + screenshot=$(echo "$counts_json" | jq -r '.failureTypes.screenshot // 0') + expectation=$(echo "$counts_json" | jq -r '.failureTypes.expectation // 0') + timeout=$(echo "$counts_json" | jq -r '.failureTypes.timeout // 0') + other=$(echo "$counts_json" | jq -r '.failureTypes.other // 0') else # Fallback parsing without jq passed=$(echo "$counts_json" | sed -n 's/.*"passed":\([0-9]*\).*/\1/p') @@ -272,13 +280,21 @@ else flaky=$(echo "$counts_json" | sed -n 's/.*"flaky":\([0-9]*\).*/\1/p') skipped=$(echo "$counts_json" | sed -n 's/.*"skipped":\([0-9]*\).*/\1/p') total=$(echo "$counts_json" | sed -n 's/.*"total":\([0-9]*\).*/\1/p') + screenshot=0 + expectation=0 + timeout=0 + other=0 fi - + total_passed=$((total_passed + ${passed:-0})) total_failed=$((total_failed + ${failed:-0})) total_flaky=$((total_flaky + ${flaky:-0})) total_skipped=$((total_skipped + ${skipped:-0})) total_tests=$((total_tests + ${total:-0})) + total_screenshot_failures=$((total_screenshot_failures + ${screenshot:-0})) + total_expectation_failures=$((total_expectation_failures + ${expectation:-0})) + total_timeout_failures=$((total_timeout_failures + ${timeout:-0})) + total_other_failures=$((total_other_failures + ${other:-0})) fi done unset IFS @@ -309,6 +325,13 @@ $status_icon **$status_text** • ⏰ $(date -u '+%m/%d/%Y, %I:%M:%S %p') UTC" comment="$comment **$total_passed** ✅ • **$total_failed** $([ $total_failed -gt 0 ] && echo '❌' || echo '✅') • **$total_flaky** $([ $total_flaky -gt 0 ] && echo '⚠️' || echo '✅') • **$total_skipped** ⏭️ • **$total_tests** total" + + # Add failure breakdown if there are failures + if [ $total_failed -gt 0 ]; then + comment="$comment + +**Failure Breakdown:** 📸 $total_screenshot_failures screenshot • ✓ $total_expectation_failures expectation • ⏱️ $total_timeout_failures timeout • ❓ $total_other_failures other" + fi fi # Collect all failing tests across browsers