mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-02-05 15:40:10 +00:00
383 lines
10 KiB
TypeScript
Executable File
383 lines
10 KiB
TypeScript
Executable File
#!/usr/bin/env tsx
|
|
import fs from 'fs'
|
|
import path from 'path'
|
|
|
|
interface TestStats {
|
|
expected?: number
|
|
unexpected?: number
|
|
flaky?: number
|
|
skipped?: number
|
|
finished?: number
|
|
}
|
|
|
|
interface TestLocation {
|
|
file: string
|
|
line: number
|
|
column: number
|
|
}
|
|
|
|
interface TestAttachment {
|
|
name: string
|
|
path?: string
|
|
contentType: string
|
|
}
|
|
|
|
interface TestResult {
|
|
status: string
|
|
duration: number
|
|
errors?: Array<{ message?: string; stack?: string }>
|
|
attachments?: TestAttachment[]
|
|
}
|
|
|
|
interface Test {
|
|
title: string
|
|
location?: TestLocation
|
|
results?: TestResult[]
|
|
}
|
|
|
|
interface Suite {
|
|
title: string
|
|
suites?: Suite[]
|
|
tests?: Test[]
|
|
}
|
|
|
|
interface ReportData {
|
|
stats?: TestStats
|
|
suites?: Suite[]
|
|
}
|
|
|
|
interface FailingTest {
|
|
name: string
|
|
filePath: string
|
|
line: number
|
|
error: string
|
|
tracePath?: string
|
|
failureType?: 'screenshot' | 'expectation' | 'timeout' | 'other'
|
|
}
|
|
|
|
interface FailureTypeCounts {
|
|
screenshot: number
|
|
expectation: number
|
|
timeout: number
|
|
other: number
|
|
}
|
|
|
|
interface TestCounts {
|
|
passed: number
|
|
failed: number
|
|
flaky: number
|
|
skipped: number
|
|
total: number
|
|
failingTests?: FailingTest[]
|
|
failureTypes?: FailureTypeCounts
|
|
}
|
|
|
|
/**
|
|
* Categorize the failure type based on error message
|
|
*/
|
|
function categorizeFailureType(
|
|
error: string,
|
|
status: string
|
|
): 'screenshot' | 'expectation' | 'timeout' | 'other' {
|
|
if (status === 'timedOut') {
|
|
return 'timeout'
|
|
}
|
|
|
|
const errorLower = error.toLowerCase()
|
|
|
|
// Screenshot-related errors
|
|
if (
|
|
errorLower.includes('screenshot') ||
|
|
errorLower.includes('snapshot') ||
|
|
errorLower.includes('toHaveScreenshot') ||
|
|
errorLower.includes('image comparison') ||
|
|
errorLower.includes('pixel') ||
|
|
errorLower.includes('visual')
|
|
) {
|
|
return 'screenshot'
|
|
}
|
|
|
|
// Expectation errors
|
|
if (
|
|
errorLower.includes('expect') ||
|
|
errorLower.includes('assertion') ||
|
|
errorLower.includes('toEqual') ||
|
|
errorLower.includes('toBe') ||
|
|
errorLower.includes('toContain') ||
|
|
errorLower.includes('toHave') ||
|
|
errorLower.includes('toMatch')
|
|
) {
|
|
return 'expectation'
|
|
}
|
|
|
|
return 'other'
|
|
}
|
|
|
|
/**
|
|
* Recursively extract failing tests from suite structure
|
|
*/
|
|
function extractFailingTests(suite: Suite, failingTests: FailingTest[]): void {
|
|
// Process tests in this suite
|
|
if (suite.tests) {
|
|
for (const test of suite.tests) {
|
|
if (!test.results) continue
|
|
|
|
for (const result of test.results) {
|
|
if (result.status === 'failed' || result.status === 'timedOut') {
|
|
const error =
|
|
result.errors?.[0]?.message ||
|
|
result.errors?.[0]?.stack ||
|
|
'Test failed'
|
|
|
|
// Find trace attachment
|
|
let tracePath: string | undefined
|
|
if (result.attachments) {
|
|
const traceAttachment = result.attachments.find(
|
|
(att) =>
|
|
att.name === 'trace' || att.contentType === 'application/zip'
|
|
)
|
|
if (traceAttachment?.path) {
|
|
tracePath = traceAttachment.path
|
|
}
|
|
}
|
|
|
|
const failureType = categorizeFailureType(error, result.status)
|
|
|
|
failingTests.push({
|
|
name: test.title,
|
|
filePath: test.location?.file || 'unknown',
|
|
line: test.location?.line || 0,
|
|
error: error.split('\n')[0], // First line of error
|
|
tracePath,
|
|
failureType
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Recursively process nested suites
|
|
if (suite.suites) {
|
|
for (const nestedSuite of suite.suites) {
|
|
extractFailingTests(nestedSuite, failingTests)
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Extract test counts from Playwright HTML report
|
|
* @param reportDir - Path to the playwright-report directory
|
|
* @returns Test counts { passed, failed, flaky, skipped, total, failingTests }
|
|
*/
|
|
function extractTestCounts(reportDir: string): TestCounts {
|
|
const counts: TestCounts = {
|
|
passed: 0,
|
|
failed: 0,
|
|
flaky: 0,
|
|
skipped: 0,
|
|
total: 0,
|
|
failingTests: [],
|
|
failureTypes: {
|
|
screenshot: 0,
|
|
expectation: 0,
|
|
timeout: 0,
|
|
other: 0
|
|
}
|
|
}
|
|
|
|
try {
|
|
// First, try to find report.json which Playwright generates with JSON reporter
|
|
const jsonReportFile = path.join(reportDir, 'report.json')
|
|
if (fs.existsSync(jsonReportFile)) {
|
|
const reportJson: ReportData = JSON.parse(
|
|
fs.readFileSync(jsonReportFile, 'utf-8')
|
|
)
|
|
if (reportJson.stats) {
|
|
const stats = reportJson.stats
|
|
counts.total = stats.expected || 0
|
|
counts.passed =
|
|
(stats.expected || 0) -
|
|
(stats.unexpected || 0) -
|
|
(stats.flaky || 0) -
|
|
(stats.skipped || 0)
|
|
counts.failed = stats.unexpected || 0
|
|
counts.flaky = stats.flaky || 0
|
|
counts.skipped = stats.skipped || 0
|
|
|
|
// Extract failing test details
|
|
if (reportJson.suites) {
|
|
for (const suite of reportJson.suites) {
|
|
extractFailingTests(suite, counts.failingTests)
|
|
}
|
|
}
|
|
|
|
// Count failure types
|
|
if (counts.failingTests) {
|
|
for (const test of counts.failingTests) {
|
|
const type = test.failureType || 'other'
|
|
counts.failureTypes![type]++
|
|
}
|
|
}
|
|
|
|
return counts
|
|
}
|
|
}
|
|
|
|
// Try index.html - Playwright HTML report embeds data in a script tag
|
|
const indexFile = path.join(reportDir, 'index.html')
|
|
if (fs.existsSync(indexFile)) {
|
|
const content = fs.readFileSync(indexFile, 'utf-8')
|
|
|
|
// Look for the embedded report data in various formats
|
|
// Format 1: window.playwrightReportBase64
|
|
let dataMatch = content.match(
|
|
/window\.playwrightReportBase64\s*=\s*["']([^"']+)["']/
|
|
)
|
|
if (dataMatch) {
|
|
try {
|
|
const decodedData = Buffer.from(dataMatch[1], 'base64').toString(
|
|
'utf-8'
|
|
)
|
|
const reportData: ReportData = JSON.parse(decodedData)
|
|
|
|
if (reportData.stats) {
|
|
const stats = reportData.stats
|
|
counts.total = stats.expected || 0
|
|
counts.passed =
|
|
(stats.expected || 0) -
|
|
(stats.unexpected || 0) -
|
|
(stats.flaky || 0) -
|
|
(stats.skipped || 0)
|
|
counts.failed = stats.unexpected || 0
|
|
counts.flaky = stats.flaky || 0
|
|
counts.skipped = stats.skipped || 0
|
|
|
|
// Extract failing test details
|
|
if (reportData.suites) {
|
|
for (const suite of reportData.suites) {
|
|
extractFailingTests(suite, counts.failingTests!)
|
|
}
|
|
}
|
|
|
|
// Count failure types
|
|
if (counts.failingTests) {
|
|
for (const test of counts.failingTests) {
|
|
const type = test.failureType || 'other'
|
|
counts.failureTypes![type]++
|
|
}
|
|
}
|
|
|
|
return counts
|
|
}
|
|
} catch (e) {
|
|
// Continue to try other formats
|
|
}
|
|
}
|
|
|
|
// Format 2: window.playwrightReport
|
|
dataMatch = content.match(/window\.playwrightReport\s*=\s*({[\s\S]*?});/)
|
|
if (dataMatch) {
|
|
try {
|
|
// Use Function constructor instead of eval for safety
|
|
const reportData = new Function(
|
|
'return ' + dataMatch[1]
|
|
)() as ReportData
|
|
|
|
if (reportData.stats) {
|
|
const stats = reportData.stats
|
|
counts.total = stats.expected || 0
|
|
counts.passed =
|
|
(stats.expected || 0) -
|
|
(stats.unexpected || 0) -
|
|
(stats.flaky || 0) -
|
|
(stats.skipped || 0)
|
|
counts.failed = stats.unexpected || 0
|
|
counts.flaky = stats.flaky || 0
|
|
counts.skipped = stats.skipped || 0
|
|
|
|
// Extract failing test details
|
|
if (reportData.suites) {
|
|
for (const suite of reportData.suites) {
|
|
extractFailingTests(suite, counts.failingTests!)
|
|
}
|
|
}
|
|
|
|
// Count failure types
|
|
if (counts.failingTests) {
|
|
for (const test of counts.failingTests) {
|
|
const type = test.failureType || 'other'
|
|
counts.failureTypes![type]++
|
|
}
|
|
}
|
|
|
|
return counts
|
|
}
|
|
} catch (e) {
|
|
// Continue to try other formats
|
|
}
|
|
}
|
|
|
|
// Format 3: Look for stats in the HTML content directly
|
|
// Playwright sometimes renders stats in the UI
|
|
const statsMatch = content.match(
|
|
/(\d+)\s+passed[^0-9]*(\d+)\s+failed[^0-9]*(\d+)\s+flaky[^0-9]*(\d+)\s+skipped/i
|
|
)
|
|
if (statsMatch) {
|
|
counts.passed = parseInt(statsMatch[1]) || 0
|
|
counts.failed = parseInt(statsMatch[2]) || 0
|
|
counts.flaky = parseInt(statsMatch[3]) || 0
|
|
counts.skipped = parseInt(statsMatch[4]) || 0
|
|
counts.total =
|
|
counts.passed + counts.failed + counts.flaky + counts.skipped
|
|
return counts
|
|
}
|
|
|
|
// Format 4: Try to extract from summary text patterns
|
|
const passedMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+passed/i)
|
|
const failedMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+failed/i)
|
|
const flakyMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+flaky/i)
|
|
const skippedMatch = content.match(/(\d+)\s+(?:tests?|specs?)\s+skipped/i)
|
|
const totalMatch = content.match(
|
|
/(\d+)\s+(?:tests?|specs?)\s+(?:total|ran)/i
|
|
)
|
|
|
|
if (passedMatch) counts.passed = parseInt(passedMatch[1]) || 0
|
|
if (failedMatch) counts.failed = parseInt(failedMatch[1]) || 0
|
|
if (flakyMatch) counts.flaky = parseInt(flakyMatch[1]) || 0
|
|
if (skippedMatch) counts.skipped = parseInt(skippedMatch[1]) || 0
|
|
if (totalMatch) {
|
|
counts.total = parseInt(totalMatch[1]) || 0
|
|
} else if (
|
|
counts.passed ||
|
|
counts.failed ||
|
|
counts.flaky ||
|
|
counts.skipped
|
|
) {
|
|
counts.total =
|
|
counts.passed + counts.failed + counts.flaky + counts.skipped
|
|
}
|
|
}
|
|
} catch (error) {
|
|
process.stderr.write(`Error reading report from ${reportDir}: ${error}\n`)
|
|
}
|
|
|
|
return counts
|
|
}
|
|
|
|
// Main execution
|
|
const reportDir = process.argv[2]
|
|
|
|
if (!reportDir) {
|
|
process.stderr.write(
|
|
'Usage: extract-playwright-counts.ts <report-directory>\n'
|
|
)
|
|
process.exit(1)
|
|
}
|
|
|
|
const counts = extractTestCounts(reportDir)
|
|
|
|
// Output as JSON for easy parsing in shell script
|
|
process.stdout.write(JSON.stringify(counts) + '\n')
|
|
|
|
export { extractTestCounts }
|