feat: add failure type categorization to Playwright PR comments

Add categorization of test failures by type (screenshot assertions,
expectation failures, timeouts, and other) to help developers quickly
understand what types of issues are occurring.

Changes:
- Add categorizeFailureType() function to detect failure types from error messages
- Track failure type counts in TestCounts interface
- Display "Failure Breakdown" section in PR comments when tests fail
- Show counts for: screenshot, expectation, timeout, and other failures

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
snomiao
2025-12-22 19:49:27 +00:00
parent 8c0c819471
commit 3af88a8b93
2 changed files with 109 additions and 3 deletions

View File

@@ -52,6 +52,14 @@ interface FailingTest {
line: number
error: string
tracePath?: string
failureType?: 'screenshot' | 'expectation' | 'timeout' | 'other'
}
interface FailureTypeCounts {
screenshot: number
expectation: number
timeout: number
other: number
}
interface TestCounts {
@@ -61,6 +69,48 @@ interface TestCounts {
skipped: number
total: number
failingTests?: FailingTest[]
failureTypes?: FailureTypeCounts
}
/**
* Categorize the failure type based on error message
*/
function categorizeFailureType(
error: string,
status: string
): 'screenshot' | 'expectation' | 'timeout' | 'other' {
if (status === 'timedOut') {
return 'timeout'
}
const errorLower = error.toLowerCase()
// Screenshot-related errors
if (
errorLower.includes('screenshot') ||
errorLower.includes('snapshot') ||
errorLower.includes('toHaveScreenshot') ||
errorLower.includes('image comparison') ||
errorLower.includes('pixel') ||
errorLower.includes('visual')
) {
return 'screenshot'
}
// Expectation errors
if (
errorLower.includes('expect') ||
errorLower.includes('assertion') ||
errorLower.includes('toEqual') ||
errorLower.includes('toBe') ||
errorLower.includes('toContain') ||
errorLower.includes('toHave') ||
errorLower.includes('toMatch')
) {
return 'expectation'
}
return 'other'
}
/**
@@ -94,12 +144,15 @@ function extractFailingTests(
}
}
const failureType = categorizeFailureType(error, result.status)
failingTests.push({
name: test.title,
filePath: test.location?.file || 'unknown',
line: test.location?.line || 0,
error: error.split('\n')[0], // First line of error
tracePath
tracePath,
failureType
})
}
}
@@ -126,7 +179,13 @@ function extractTestCounts(reportDir: string): TestCounts {
flaky: 0,
skipped: 0,
total: 0,
failingTests: []
failingTests: [],
failureTypes: {
screenshot: 0,
expectation: 0,
timeout: 0,
other: 0
}
}
try {
@@ -155,6 +214,14 @@ function extractTestCounts(reportDir: string): TestCounts {
}
}
// Count failure types
if (counts.failingTests) {
for (const test of counts.failingTests) {
const type = test.failureType || 'other'
counts.failureTypes![type]++
}
}
return counts
}
}
@@ -195,6 +262,14 @@ function extractTestCounts(reportDir: string): TestCounts {
}
}
// Count failure types
if (counts.failingTests) {
for (const test of counts.failingTests) {
const type = test.failureType || 'other'
counts.failureTypes![type]++
}
}
return counts
}
} catch (e) {
@@ -230,6 +305,14 @@ function extractTestCounts(reportDir: string): TestCounts {
}
}
// Count failure types
if (counts.failingTests) {
for (const test of counts.failingTests) {
const type = test.failureType || 'other'
counts.failureTypes![type]++
}
}
return counts
}
} catch (e) {