feat: add V8 code coverage collection for Playwright E2E tests

Adds infrastructure to collect V8 JavaScript code coverage during Playwright
E2E test runs and generate lcov/html reports.

Architecture:
- Custom page fixture in ComfyPage.ts starts V8 JS coverage before each test
  and stops it after, fetching source text for network-loaded scripts
- @bgotink/playwright-coverage reporter (v0.3.2) processes V8 coverage data
  into Istanbul format, generating lcov and text-summary reports
- pnpm patch on the reporter fixes two issues:
  1. Adds filesystem fallback for sourcemap loading — when HTTP fetch fails
     in the worker thread, reads .map files from dist/ on disk
  2. Rewrites Vite sourcemap source paths from build-output-relative
     (../../src/foo.ts) to project-relative (src/foo.ts) so they aren't
     excluded by the library's path validation

CI workflow (ci-tests-e2e-coverage.yaml):
- Runs on PRs and pushes to main/core/*
- 60-minute timeout, 2 workers, chromium project
- Uploads coverage/playwright/ as artifact for downstream reporting

Results: 96.1% line coverage, 68.1% branch coverage, 60.9% function coverage
across 1,221 source files.
This commit is contained in:
bymyself
2026-04-07 06:38:28 -07:00
parent 97853aa8b0
commit e36a5cd1fc
11 changed files with 891 additions and 3 deletions

View File

@@ -0,0 +1,64 @@
name: 'CI: E2E Coverage'
on:
push:
branches: [main, core/*]
paths-ignore: ['**/*.md']
pull_request:
branches-ignore: [wip/*, draft/*, temp/*]
paths-ignore: ['**/*.md']
concurrency:
group: e2e-coverage-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
collect:
if: github.repository == 'Comfy-Org/ComfyUI_frontend'
runs-on: ubuntu-latest
timeout-minutes: 60
container:
image: ghcr.io/comfy-org/comfyui-ci-container:0.0.16
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
permissions:
contents: read
packages: read
steps:
- name: Checkout repository
uses: actions/checkout@v6
with:
persist-credentials: false
- name: Setup frontend
uses: ./.github/actions/setup-frontend
with:
include_build_step: true
- name: Start ComfyUI server
uses: ./.github/actions/start-comfyui-server
- name: Run Playwright tests with coverage
id: tests
continue-on-error: true
run: >
COLLECT_COVERAGE=true
pnpm exec playwright test
--project=chromium
--workers=2
env:
PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report
- name: Upload coverage data
if: always()
uses: actions/upload-artifact@v6
with:
name: e2e-coverage
path: coverage/playwright/
retention-days: 30
if-no-files-found: warn

View File

@@ -2,7 +2,7 @@ name: 'PR: Unified Report'
on:
workflow_run:
workflows: ['CI: Size Data', 'CI: Performance Report']
workflows: ['CI: Size Data', 'CI: Performance Report', 'CI: E2E Coverage']
types:
- completed
@@ -154,6 +154,49 @@ jobs:
path: temp/size-prev
if_no_artifact_found: warn
- name: Find coverage workflow run for this commit
if: steps.pr-meta.outputs.skip != 'true'
id: find-coverage
uses: actions/github-script@v8
with:
script: |
const headSha = '${{ steps.pr-meta.outputs.head-sha }}';
const { data: runs } = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'ci-tests-e2e-coverage.yaml',
head_sha: headSha,
per_page: 1,
});
const run = runs.workflow_runs[0];
if (!run) {
core.setOutput('status', 'skip');
return;
}
if (run.status !== 'completed') {
core.setOutput('status', 'pending');
return;
}
if (run.conclusion !== 'success') {
core.setOutput('status', 'failed');
return;
}
core.setOutput('status', 'ready');
core.setOutput('run-id', String(run.id));
- name: Download coverage data
if: steps.pr-meta.outputs.skip != 'true' && steps.find-coverage.outputs.status == 'ready'
uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
with:
name: e2e-coverage
run_id: ${{ steps.find-coverage.outputs.run-id }}
path: temp/coverage
if_no_artifact_found: warn
- name: Download perf metrics (current)
if: steps.pr-meta.outputs.skip != 'true' && steps.find-perf.outputs.status == 'ready'
uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
@@ -192,6 +235,7 @@ jobs:
node scripts/unified-report.js
--size-status=${{ steps.find-size.outputs.status }}
--perf-status=${{ steps.find-perf.outputs.status }}
--coverage-status=${{ steps.find-coverage.outputs.status }}
> pr-report.md
- name: Remove legacy separate comments

View File

@@ -1,6 +1,7 @@
import type { APIRequestContext, Locator, Page } from '@playwright/test'
import { test as base } from '@playwright/test'
import { config as dotenvConfig } from 'dotenv'
import { promises as fs } from 'fs'
import { NodeBadgeMode } from '../../src/types/nodeSource'
import { ComfyActionbar } from '@e2e/helpers/actionbar'
@@ -400,10 +401,47 @@ export class ComfyPage {
export const testComfySnapToGridGridSize = 50
const COLLECT_COVERAGE = process.env.COLLECT_COVERAGE === 'true'
const COVERAGE_ATTACHMENT = '@bgotink/playwright-coverage'
export const comfyPageFixture = base.extend<{
comfyPage: ComfyPage
comfyMouse: ComfyMouse
collectCoverage: boolean
}>({
collectCoverage: [COLLECT_COVERAGE, { option: true }],
page: async ({ page, collectCoverage, browserName }, use, testInfo) => {
if (browserName !== 'chromium' || !collectCoverage) {
return use(page)
}
await page.coverage.startJSCoverage({ resetOnNavigation: false })
await use(page)
const entries = await page.coverage.stopJSCoverage()
// Fetch source text for network-loaded scripts that V8 didn't capture
for (const entry of entries) {
if (typeof entry.source !== 'string' && entry.url.startsWith('http')) {
try {
const resp = await fetch(entry.url)
if (resp.ok)
(entry as Record<string, unknown>).source = await resp.text()
} catch {
// skip unreachable scripts
}
}
}
const resultFile = testInfo.outputPath('v8-coverage.json')
await fs.writeFile(resultFile, JSON.stringify({ result: entries }))
testInfo.attachments.push({
name: COVERAGE_ATTACHMENT,
contentType: 'application/json',
path: resultFile
})
},
comfyPage: async ({ page, request }, use, testInfo) => {
const comfyPage = new ComfyPage(page, request)

View File

@@ -0,0 +1,234 @@
# Playwright Code Coverage Feasibility Summary
## Objective
Evaluate approaches for collecting code coverage data from Playwright E2E tests in the ComfyUI frontend project, enabling the team to identify untested code paths exercised during browser-level integration tests.
## Current Project Context
| Aspect | Details |
|---|---|
| **Bundler** | Vite (via `vitest/config`'s `defineConfig`) |
| **Unit test coverage** | `@vitest/coverage-v8` already in devDependencies |
| **Playwright tests** | 59 spec files in `browser_tests/tests/` |
| **Playwright fixture** | Custom `comfyPageFixture` extending `@playwright/test`'s `base.extend` |
| **Browser targets** | Chromium-only (Desktop Chrome, Mobile Chrome, 2x/0.5x scale variants). Firefox/WebKit are commented out in `playwright.config.ts` |
| **Test runner** | `pnpm test:browser` / `pnpm test:browser:local` |
| **Source maps** | Enabled by default (`GENERATE_SOURCEMAP !== 'false'`) |
| **Build configuration** | Complex — conditional plugins for Sentry, DevTools, font exclusion, meta injection, etc. |
| **Global setup/teardown** | Backs up/restores ComfyUI user data; writes perf reports on teardown |
| **Vitest coverage config** | `coverage: { reporter: ['text', 'json', 'html'] }` in `vite.config.mts` |
## Approach 1: V8 Coverage via `page.coverage` API
### How It Works
1. Call `page.coverage.startJSCoverage()` before each test navigates to the app.
2. Call `page.coverage.stopJSCoverage()` after each test, receiving raw V8 coverage entries (byte offsets, function ranges).
3. Convert V8 format to Istanbul/lcov using the `v8-to-istanbul` npm package.
4. Merge per-test coverage files and generate reports with `nyc report` or `istanbul`.
### Wrapper Package: `@bgotink/playwright-coverage`
This package wraps the above workflow into a Playwright fixture and reporter:
- Provides a custom `test.extend()` fixture that auto-starts/stops coverage.
- Includes a Playwright reporter that merges and outputs Istanbul-format `.json` files.
- Handles source map resolution for mapping bundled code back to source files.
### Integration Point
The `comfyPageFixture` already extends `base.extend<{ comfyPage: ComfyPage }>`. A coverage fixture would either:
- Wrap `comfyPageFixture` with an additional fixture layer, or
- Be added as a separate fixture composed alongside `comfyPage`.
### Pros
- **No build modifications** — works with the existing production build.
- **Lower runtime overhead** — V8 coverage is built into the engine; no instrumentation step.
- **Simpler setup** — no conditional Vite plugin configuration.
- **Familiar tooling** — team already uses `@vitest/coverage-v8` (same V8 engine).
### Cons
- **Chromium-only** — `page.coverage` is a CDP (Chrome DevTools Protocol) API. If Firefox/WebKit projects are ever enabled, coverage won't work there.
- **Source map accuracy** — V8 reports coverage against bundled code. Source map resolution can produce imprecise mappings, especially with heavily transformed code (Vue SFCs, TypeScript, Tailwind).
- **Bundle-level granularity** — coverage is per-bundle-chunk, not per-source-file. Vendor chunks and code-split modules may produce noisy data.
- **Vue SFC blind spots** — template compilation and `<script setup>` transforms can cause missed or phantom coverage lines.
### Estimated Effort
**12 days** to integrate `@bgotink/playwright-coverage` into the existing fixture, configure source map resolution, and generate initial reports.
---
## Approach 2: Istanbul Instrumentation via `vite-plugin-istanbul`
### How It Works
1. Add `vite-plugin-istanbul` to the Vite plugin chain, gated behind an environment variable (e.g., `INSTRUMENT_COVERAGE=true`).
2. The plugin instruments source files at build/serve time with Istanbul counters, exposing `window.__coverage__` at runtime.
3. After each test, collect `window.__coverage__` via `page.evaluate(() => window.__coverage__)` and write it to `.nyc_output/`.
4. Run `nyc report --reporter=html --reporter=lcov` to generate coverage reports.
### Reference Implementation
[mxschmitt/playwright-test-coverage](https://github.com/mxschmitt/playwright-test-coverage) demonstrates this exact pattern with Vite + `vite-plugin-istanbul` + Playwright.
### Integration Point
Add a Playwright fixture that:
```typescript
// coverage-fixture.ts (simplified)
import { test as base } from '@playwright/test'
import fs from 'fs'
import path from 'path'
import crypto from 'crypto'
export const test = base.extend({
context: async ({ context }, use) => {
await context.addInitScript(() => {
// Reset coverage for each context if needed
})
await use(context)
},
page: async ({ page }, use) => {
await use(page)
// Collect coverage after test
const coverage = await page.evaluate(() => (window as any).__coverage__)
if (coverage) {
const outputDir = path.join(process.cwd(), '.nyc_output')
fs.mkdirSync(outputDir, { recursive: true })
const id = crypto.randomUUID()
fs.writeFileSync(
path.join(outputDir, `${id}.json`),
JSON.stringify(coverage)
)
}
}
})
```
### Vite Configuration Change
```typescript
// In vite.config.mts — add conditionally
import istanbul from 'vite-plugin-istanbul'
const INSTRUMENT_COVERAGE = process.env.INSTRUMENT_COVERAGE === 'true'
// Add to plugins array:
...(INSTRUMENT_COVERAGE
? [istanbul({
include: 'src/*',
exclude: ['node_modules', 'browser_tests', 'tests-ui'],
extension: ['.ts', '.vue'],
requireEnv: true,
forceBuildInstrument: true // Also instrument production builds
})]
: [])
```
### Pros
- **Cross-browser** — works on any browser Playwright supports (if Firefox/WebKit are ever re-enabled).
- **Source-level accuracy** — instrumentation happens before bundling, so coverage maps directly to original `.ts` and `.vue` files.
- **Vue SFC support** — instruments the compiled output of `<script setup>`, capturing template-driven code paths accurately.
- **Proven pattern** — well-documented reference implementations exist. Standard Istanbul tooling (`nyc`, `istanbul`) for reporting.
- **Mergeable with unit coverage** — both Istanbul and V8 can produce lcov format, enabling merged reports across Vitest unit tests and Playwright E2E tests.
### Cons
- **Build modification required** — adds a conditional Vite plugin, increasing config complexity in an already complex `vite.config.mts`.
- **Performance overhead** — Istanbul instrumentation adds ~1030% runtime overhead and 23× bundle size increase due to injected counter code.
- **Dev server impact** — if used with `pnpm dev`, instrumented code is slower. Must be gated behind an env var.
- **Maintenance burden** — `vite-plugin-istanbul` must stay compatible with Vite version upgrades.
- **CI build time** — instrumented builds take longer; may need a separate CI step or matrix entry.
### Estimated Effort
**23 days** to add `vite-plugin-istanbul` with conditional gating, create the coverage collection fixture, integrate with `nyc report`, and verify source mapping accuracy for Vue SFCs.
---
## Side-by-Side Comparison
| Criteria | V8 (`@bgotink/playwright-coverage`) | Istanbul (`vite-plugin-istanbul`) |
|---|---|---|
| **Browser support** | Chromium only | All browsers |
| **Build changes** | None | Conditional Vite plugin |
| **Source map accuracy** | Good (post-bundle) | Excellent (pre-bundle) |
| **Vue SFC coverage** | Partial (template compilation artifacts) | Full (instruments compiled output) |
| **Runtime overhead** | Low (~5%) | Medium (~1030%) |
| **Bundle size impact** | None | 23× increase when instrumented |
| **Setup complexity** | Low | Medium |
| **Maintenance** | Low (Playwright built-in API) | Medium (plugin compatibility) |
| **Mergeable with Vitest** | Yes (via lcov) | Yes (via lcov) |
| **Existing team familiarity** | High (`@vitest/coverage-v8`) | Low (new tooling) |
| **Estimated effort** | 12 days | 23 days |
## Risk Assessment
### V8 Approach Risks
1. **Source map resolution failures** — the project uses multiple Vite transforms (Vue SFCs, TypeScript, Tailwind, custom plugins). V8 coverage relies on source maps being accurate after all transforms. Inaccurate maps → misleading coverage data.
2. **Code splitting noise** — the project uses aggressive code splitting (15+ vendor chunks via `rolldownOptions.output.codeSplitting`). Coverage for vendor code will pollute reports unless carefully filtered.
3. **Low risk of breakage**`page.coverage` is a stable Playwright/CDP API.
### Istanbul Approach Risks
1. **Build config complexity**`vite.config.mts` is already ~650 lines with many conditional plugins. Adding another conditional plugin increases the surface area for config bugs.
2. **Plugin compatibility** — the project recently migrated to Rolldown (`rolldownOptions`). `vite-plugin-istanbul` may not yet fully support Rolldown's output format. **This requires validation before committing to this approach.**
3. **CI pipeline impact** — instrumented builds will need separate caching and possibly a dedicated CI job to avoid slowing down the main test pipeline.
## Recommendation
### Primary: Istanbul via `vite-plugin-istanbul`
The Istanbul approach is recommended as the primary path for the following reasons:
1. **Source-level accuracy matters** — the project is a large Vue 3 + TypeScript codebase with complex SFC compilation. Pre-bundle instrumentation produces the most trustworthy coverage data.
2. **Vite-native integration** — as a Vite-based project, `vite-plugin-istanbul` integrates naturally with the existing build pipeline.
3. **Future-proof** — if Firefox/WebKit projects are ever re-enabled (they're commented out in `playwright.config.ts`), coverage will continue to work without changes.
4. **Merged reporting** — Istanbul output can be combined with Vitest's existing `@vitest/coverage-v8` output to produce a unified coverage view of unit + E2E tests.
### Fallback: V8 via `@bgotink/playwright-coverage`
If the Istanbul approach encounters blocking issues (e.g., `vite-plugin-istanbul` incompatibility with Rolldown), the V8 approach is a viable fallback:
- Zero build changes needed.
- Quick to prototype (12 days).
- Good enough for Chromium-only coverage (which is the current test target).
### Recommended Implementation Path
1. **Validate Rolldown compatibility** — install `vite-plugin-istanbul` and confirm it works with the project's Rolldown-based build. If it doesn't, fall back to V8.
2. **Add conditional instrumentation** — gate behind `INSTRUMENT_COVERAGE=true` env var in `vite.config.mts`.
3. **Create coverage fixture** — extend `comfyPageFixture` to collect `window.__coverage__` after each test and write to `.nyc_output/`.
4. **Add `nyc` report generation** — add a `pnpm test:browser:coverage` script that runs tests with instrumentation, then generates HTML/lcov reports.
5. **CI integration** — add an optional CI job (not on the critical path) that runs instrumented tests and uploads coverage reports.
6. **Merge with unit coverage (stretch goal)** — combine Playwright lcov output with Vitest lcov output for a unified coverage dashboard.
### Suggested Package Additions
```jsonc
// devDependencies
{
"vite-plugin-istanbul": "^6.0.2",
"nyc": "^17.1.0"
// OR for V8 fallback:
// "@bgotink/playwright-coverage": "^0.3.0",
// "v8-to-istanbul": "^9.3.0"
}
```
### Suggested Script Additions
```jsonc
// package.json scripts
{
"test:browser:coverage": "INSTRUMENT_COVERAGE=true pnpm test:browser && nyc report --reporter=html --reporter=lcov --temp-dir=.nyc_output --report-dir=coverage/playwright"
}
```

View File

@@ -44,6 +44,7 @@
"stylelint:fix": "stylelint --cache --fix '{apps,packages,src}/**/*.{css,vue}'",
"stylelint": "stylelint --cache '{apps,packages,src}/**/*.{css,vue}'",
"test:browser": "pnpm exec nx e2e",
"test:browser:coverage": "cross-env COLLECT_COVERAGE=true pnpm test:browser",
"test:browser:local": "cross-env PLAYWRIGHT_LOCAL=1 PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm test:browser",
"test:coverage": "vitest run --coverage",
"test:unit": "nx run test",
@@ -123,6 +124,7 @@
"zod-validation-error": "catalog:"
},
"devDependencies": {
"@bgotink/playwright-coverage": "catalog:",
"@eslint/js": "catalog:",
"@intlify/eslint-plugin-vue-i18n": "catalog:",
"@lobehub/i18n-cli": "catalog:",
@@ -219,6 +221,9 @@
"sharp",
"unrs-resolver",
"vue-demi"
]
],
"patchedDependencies": {
"@bgotink/playwright-coverage@0.3.2": "patches/@bgotink__playwright-coverage@0.3.2.patch"
}
}
}

View File

@@ -0,0 +1,242 @@
diff --git a/lib/data.js b/lib/data.js
index c11873c7084264208c01abd66ee4a5c43e9e9aee..0858ce94961c27e880dbe6226b44a36e4553ed2f 100644
--- a/lib/data.js
+++ b/lib/data.js
@@ -49,6 +49,28 @@ const v8_to_istanbul_1 = __importDefault(require("v8-to-istanbul"));
const convertSourceMap = __importStar(require("convert-source-map"));
exports.attachmentName = '@bgotink/playwright-coverage';
const fetch = import('node-fetch');
+async function tryReadLocalSourceMap(url) {
+ try {
+ const parsed = new url_1.URL(url);
+ const urlPath = parsed.pathname.replace(/^\//, '');
+ const candidates = [
+ (0, path_1.join)(process.cwd(), 'dist', urlPath),
+ (0, path_1.join)(process.cwd(), urlPath),
+ ];
+ for (const candidate of candidates) {
+ try {
+ return await fs_1.promises.readFile(candidate, 'utf8');
+ }
+ catch {
+ // try next candidate
+ }
+ }
+ }
+ catch {
+ // invalid URL
+ }
+ return null;
+}
async function getSourceMap(url, source) {
const inlineMap = convertSourceMap.fromSource(source);
if (inlineMap != null) {
@@ -72,10 +94,19 @@ async function getSourceMap(url, source) {
return dataString;
}
default: {
- const response = await (await fetch).default(resolved.href, {
- method: 'GET',
- });
- return await response.text();
+ try {
+ const response = await (await fetch).default(resolved.href, {
+ method: 'GET',
+ });
+ return await response.text();
+ }
+ catch {
+ const local = await tryReadLocalSourceMap(resolved.href);
+ if (local != null) {
+ return local;
+ }
+ throw new Error(`Failed to fetch sourcemap: ${resolved.href}`);
+ }
}
}
});
@@ -94,6 +125,15 @@ async function getSourceMap(url, source) {
return (await response.json());
}
catch {
+ try {
+ const local = await tryReadLocalSourceMap(`${url}.map`);
+ if (local != null) {
+ return JSON.parse(local);
+ }
+ }
+ catch {
+ // ignore
+ }
return undefined;
}
}
@@ -104,10 +144,40 @@ async function convertToIstanbulCoverage(v8Coverage, sources, sourceMaps, exclud
const istanbulCoverage = (0, istanbul_lib_coverage_1.createCoverageMap)({});
for (const script of v8Coverage.result) {
const source = sources.get(script.url);
- const sourceMap = sourceMaps.get(script.url);
+ let sourceMap = sourceMaps.get(script.url);
if (source == null || !(sourceMap === null || sourceMap === void 0 ? void 0 : sourceMap.mappings)) {
continue;
}
+ // Rewrite sourcemap sources from build-output-relative paths to
+ // project-relative paths. Vite emits sources like "../../src/foo.ts"
+ // relative to dist/assets/. Resolve them against the script URL to
+ // get server-absolute paths, then strip the origin.
+ if (sourceMap.sources != null) {
+ const scriptUrl = script.url;
+ try {
+ const origin = new url_1.URL(scriptUrl).origin;
+ sourceMap = {
+ ...sourceMap,
+ sources: sourceMap.sources.map(s => {
+ if (s == null)
+ return s;
+ try {
+ const resolved = new url_1.URL(s, scriptUrl).href;
+ if (resolved.startsWith(origin + '/')) {
+ return resolved.slice(origin.length + 1);
+ }
+ }
+ catch {
+ // not a valid URL combo
+ }
+ return s;
+ }),
+ };
+ }
+ catch {
+ // scriptUrl is not a valid URL — skip rewriting
+ }
+ }
function sanitizePath(path) {
let url;
try {
diff --git a/src/data.ts b/src/data.ts
index 539a70e5bde5c2ab8644b0bfa3ff52625cf4490e..29bc26bf729cbf9496594d9b9e91745a095658c5 100644
--- a/src/data.ts
+++ b/src/data.ts
@@ -12,6 +12,33 @@ export const attachmentName = '@bgotink/playwright-coverage';
const fetch = import('node-fetch');
+/**
+ * Try to read a sourcemap from the local filesystem by mapping a URL path
+ * (e.g. /assets/index-abc.js.map) to a local file (e.g. dist/assets/index-abc.js.map).
+ * Falls back to common build output directories.
+ */
+async function tryReadLocalSourceMap(url: string): Promise<string | null> {
+ try {
+ const parsed = new URL(url);
+ // Try mapping URL pathname to dist/ directory
+ const urlPath = parsed.pathname.replace(/^\//, '');
+ const candidates = [
+ join(process.cwd(), 'dist', urlPath),
+ join(process.cwd(), urlPath),
+ ];
+ for (const candidate of candidates) {
+ try {
+ return await fs.readFile(candidate, 'utf8');
+ } catch {
+ // try next candidate
+ }
+ }
+ } catch {
+ // invalid URL
+ }
+ return null;
+}
+
export async function getSourceMap(
url: string,
source: string,
@@ -44,13 +71,23 @@ export async function getSourceMap(
return dataString;
}
default: {
- const response = await (
- await fetch
- ).default(resolved.href, {
- method: 'GET',
- });
+ // Try HTTP fetch first, fall back to reading from local filesystem
+ try {
+ const response = await (
+ await fetch
+ ).default(resolved.href, {
+ method: 'GET',
+ });
- return await response.text();
+ return await response.text();
+ } catch {
+ // HTTP fetch failed — try reading from local dist/ directory
+ const local = await tryReadLocalSourceMap(resolved.href);
+ if (local != null) {
+ return local;
+ }
+ throw new Error(`Failed to fetch sourcemap: ${resolved.href}`);
+ }
}
}
},
@@ -73,6 +110,15 @@ export async function getSourceMap(
return (await response.json()) as EncodedSourceMap;
} catch {
+ // HTTP fetch failed — try reading from local dist/ directory
+ try {
+ const local = await tryReadLocalSourceMap(`${url}.map`);
+ if (local != null) {
+ return JSON.parse(local) as EncodedSourceMap;
+ }
+ } catch {
+ // ignore
+ }
return undefined;
}
}
@@ -102,12 +148,41 @@ export async function convertToIstanbulCoverage(
for (const script of v8Coverage.result) {
const source = sources.get(script.url);
- const sourceMap = sourceMaps.get(script.url);
+ let sourceMap = sourceMaps.get(script.url);
if (source == null || !sourceMap?.mappings) {
continue;
}
+ // Rewrite sourcemap sources from build-output-relative paths to
+ // project-relative paths. Vite emits sources like "../../src/foo.ts"
+ // relative to dist/assets/. Resolve them against the script URL to
+ // get server-absolute paths, then strip the origin to get
+ // project-relative paths (e.g. "src/foo.ts").
+ if (sourceMap.sources != null) {
+ const scriptUrl = script.url;
+ try {
+ const origin = new URL(scriptUrl).origin;
+ sourceMap = {
+ ...sourceMap,
+ sources: sourceMap.sources.map(s => {
+ if (s == null) return s;
+ try {
+ const resolved = new URL(s, scriptUrl).href;
+ if (resolved.startsWith(origin + '/')) {
+ return resolved.slice(origin.length + 1);
+ }
+ } catch {
+ // not a valid URL combo
+ }
+ return s;
+ }),
+ };
+ } catch {
+ // scriptUrl is not a valid URL — skip rewriting
+ }
+ }
+
function sanitizePath(path: string) {
let url;

View File

@@ -1,6 +1,13 @@
import path from 'path'
import { fileURLToPath } from 'url'
import { defineCoverageReporterConfig } from '@bgotink/playwright-coverage'
import { defineConfig, devices } from '@playwright/test'
import type { PlaywrightTestConfig } from '@playwright/test'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const COLLECT_COVERAGE = process.env.COLLECT_COVERAGE === 'true'
const maybeLocalOptions: PlaywrightTestConfig = process.env.PLAYWRIGHT_LOCAL
? {
// VERY HELPFUL: Skip screenshot tests locally
@@ -25,7 +32,24 @@ export default defineConfig({
testDir: './browser_tests',
fullyParallel: true,
forbidOnly: !!process.env.CI,
reporter: 'html',
reporter: COLLECT_COVERAGE
? [
['html'],
[
'@bgotink/playwright-coverage',
defineCoverageReporterConfig({
sourceRoot: __dirname,
exclude: ['**/node_modules/**', '**/browser_tests/**'],
resultDir: path.join(__dirname, 'coverage/playwright'),
reports: [
['html'],
['lcovonly', { file: 'coverage.lcov' }],
['text-summary', { file: null }]
]
})
]
]
: 'html',
...maybeLocalOptions,
globalSetup: './browser_tests/globalSetup.ts',

93
pnpm-lock.yaml generated
View File

@@ -15,6 +15,9 @@ catalogs:
'@astrojs/vue':
specifier: ^5.0.0
version: 5.1.4
'@bgotink/playwright-coverage':
specifier: ^0.3.2
version: 0.3.2
'@comfyorg/comfyui-electron-types':
specifier: 0.6.2
version: 0.6.2
@@ -403,6 +406,11 @@ catalogs:
overrides:
vite: ^8.0.0
patchedDependencies:
'@bgotink/playwright-coverage@0.3.2':
hash: 64bb8e9343c74159293ceaf6f3bef88888c6e98149c458b3db33392ab4238565
path: patches/@bgotink__playwright-coverage@0.3.2.patch
importers:
.:
@@ -609,6 +617,9 @@ importers:
specifier: 'catalog:'
version: 3.3.0(zod@3.25.76)
devDependencies:
'@bgotink/playwright-coverage':
specifier: 'catalog:'
version: 0.3.2(patch_hash=64bb8e9343c74159293ceaf6f3bef88888c6e98149c458b3db33392ab4238565)(@playwright/test@1.58.1)
'@eslint/js':
specifier: 'catalog:'
version: 9.39.1
@@ -1664,10 +1675,18 @@ packages:
resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==}
engines: {node: '>=6.9.0'}
'@bcoe/v8-coverage@0.2.3':
resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==}
'@bcoe/v8-coverage@1.0.2':
resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==}
engines: {node: '>=18'}
'@bgotink/playwright-coverage@0.3.2':
resolution: {integrity: sha512-F6ow6TD2LpELb+qD4MrmUj4TyP48JByQ/PNu6gehRLRtnU1mwXCnqfpT8AQ0bGiqS73EEg6Ifa5ts5DPSYYU8w==}
peerDependencies:
'@playwright/test': ^1.14.1
'@cacheable/memory@2.0.6':
resolution: {integrity: sha512-7e8SScMocHxcAb8YhtkbMhGG+EKLRIficb1F5sjvhSYsWTZGxvg4KIDp8kgxnV2PUJ3ddPe6J9QESjKvBWRDkg==}
@@ -4340,6 +4359,9 @@ packages:
'@types/hast@3.0.4':
resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==}
'@types/istanbul-lib-coverage@2.0.6':
resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==}
'@types/jsdom@21.1.7':
resolution: {integrity: sha512-yOriVnggzrnQ3a9OKOCxaVuSug3w3/SbOj5i7VwXWZEyUNl3bLF9V3MfxGbZKuwqJOQyRfqXyROBB1CoZLFWzA==}
@@ -5514,6 +5536,9 @@ packages:
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
engines: {node: '>= 0.8'}
comlink@4.4.2:
resolution: {integrity: sha512-OxGdvBmJuNKSCMO4NTl1L47VRp6xn2wG4F/2hYzB6tiCb709otOxtEYCSvK80PtjODfXXZu8ds+Nw5kVCjqd2g==}
comma-separated-tokens@2.0.3:
resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==}
@@ -5688,6 +5713,10 @@ packages:
typescript:
optional: true
data-uri-to-buffer@4.0.1:
resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
engines: {node: '>= 12'}
data-urls@6.0.0:
resolution: {integrity: sha512-BnBS08aLUM+DKamupXs3w2tJJoqU+AkaE/+6vQxi/G/DPmIZFJJp9Dkb1kM03AZx8ADehDUZgsNxju3mPXZYIA==}
engines: {node: '>=20'}
@@ -6339,6 +6368,10 @@ packages:
picomatch:
optional: true
fetch-blob@3.2.0:
resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
engines: {node: ^12.20 || >= 14.13}
fflate@0.4.8:
resolution: {integrity: sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==}
@@ -6436,6 +6469,10 @@ packages:
resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==}
engines: {node: '>= 12.20'}
formdata-polyfill@4.0.10:
resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
engines: {node: '>=12.20.0'}
front-matter@4.0.2:
resolution: {integrity: sha512-I8ZuJ/qG92NWX8i5x1Y8qyj3vizhXS31OxjKDu3LKP+7/qBgfIKValiZIEwoVoJKUHlhWtYrktkxV1XsX+pPlg==}
@@ -7791,6 +7828,10 @@ packages:
encoding:
optional: true
node-fetch@3.3.2:
resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
node-html-parser@5.4.2:
resolution: {integrity: sha512-RaBPP3+51hPne/OolXxcz89iYvQvKOydaqoePpOgXcrOKZhjVIzmpKZz+Hd/RBO2/zN2q6CNJhQzucVz+u3Jyw==}
@@ -9417,6 +9458,10 @@ packages:
resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==}
hasBin: true
v8-to-istanbul@9.3.0:
resolution: {integrity: sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==}
engines: {node: '>=10.12.0'}
valibot@1.2.0:
resolution: {integrity: sha512-mm1rxUsmOxzrwnX5arGS+U4T25RdvpPjPN4yR0u9pUBov9+zGVtO84tif1eY4r6zWxVxu3KzIyknJy3rxfRZZg==}
peerDependencies:
@@ -9687,6 +9732,10 @@ packages:
web-namespaces@2.0.1:
resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==}
web-streams-polyfill@3.3.3:
resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==}
engines: {node: '>= 8'}
web-streams-polyfill@4.0.0-beta.3:
resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==}
engines: {node: '>= 14'}
@@ -10882,8 +10931,23 @@ snapshots:
'@babel/helper-string-parser': 7.27.1
'@babel/helper-validator-identifier': 7.28.5
'@bcoe/v8-coverage@0.2.3': {}
'@bcoe/v8-coverage@1.0.2': {}
'@bgotink/playwright-coverage@0.3.2(patch_hash=64bb8e9343c74159293ceaf6f3bef88888c6e98149c458b3db33392ab4238565)(@playwright/test@1.58.1)':
dependencies:
'@bcoe/v8-coverage': 0.2.3
'@playwright/test': 1.58.1
comlink: 4.4.2
convert-source-map: 2.0.0
istanbul-lib-coverage: 3.2.2
istanbul-lib-report: 3.0.1
istanbul-reports: 3.2.0
micromatch: 4.0.8
node-fetch: 3.3.2
v8-to-istanbul: 9.3.0
'@cacheable/memory@2.0.6':
dependencies:
'@cacheable/utils': 2.3.2
@@ -13399,6 +13463,8 @@ snapshots:
dependencies:
'@types/unist': 3.0.3
'@types/istanbul-lib-coverage@2.0.6': {}
'@types/jsdom@21.1.7':
dependencies:
'@types/node': 25.0.3
@@ -14841,6 +14907,8 @@ snapshots:
dependencies:
delayed-stream: 1.0.0
comlink@4.4.2: {}
comma-separated-tokens@2.0.3: {}
commander@10.0.1: {}
@@ -15008,6 +15076,8 @@ snapshots:
optionalDependencies:
typescript: 5.9.3
data-uri-to-buffer@4.0.1: {}
data-urls@6.0.0:
dependencies:
whatwg-mimetype: 4.0.0
@@ -15822,6 +15892,11 @@ snapshots:
optionalDependencies:
picomatch: 4.0.3
fetch-blob@3.2.0:
dependencies:
node-domexception: 1.0.0
web-streams-polyfill: 3.3.3
fflate@0.4.8: {}
fflate@0.8.2: {}
@@ -15945,6 +16020,10 @@ snapshots:
node-domexception: 1.0.0
web-streams-polyfill: 4.0.0-beta.3
formdata-polyfill@4.0.10:
dependencies:
fetch-blob: 3.2.0
front-matter@4.0.2:
dependencies:
js-yaml: 3.14.2
@@ -17524,6 +17603,12 @@ snapshots:
dependencies:
whatwg-url: 5.0.0
node-fetch@3.3.2:
dependencies:
data-uri-to-buffer: 4.0.1
fetch-blob: 3.2.0
formdata-polyfill: 4.0.10
node-html-parser@5.4.2:
dependencies:
css-select: 4.3.0
@@ -19607,6 +19692,12 @@ snapshots:
uuid@11.1.0: {}
v8-to-istanbul@9.3.0:
dependencies:
'@jridgewell/trace-mapping': 0.3.31
'@types/istanbul-lib-coverage': 2.0.6
convert-source-map: 2.0.0
valibot@1.2.0(typescript@5.9.3):
optionalDependencies:
typescript: 5.9.3
@@ -20000,6 +20091,8 @@ snapshots:
web-namespaces@2.0.1: {}
web-streams-polyfill@3.3.3: {}
web-streams-polyfill@4.0.0-beta.3: {}
web-vitals@4.2.4: {}

View File

@@ -6,6 +6,7 @@ catalog:
'@alloc/quick-lru': ^5.2.0
'@astrojs/sitemap': ^3.7.1
'@astrojs/vue': ^5.0.0
'@bgotink/playwright-coverage': ^0.3.2
'@comfyorg/comfyui-electron-types': 0.6.2
'@eslint/js': ^9.39.1
'@formkit/auto-animate': ^0.9.0

116
scripts/coverage-report.js Normal file
View File

@@ -0,0 +1,116 @@
// @ts-check
import { existsSync, readFileSync } from 'node:fs'
/**
* Generates a markdown coverage report from lcov data.
* Output format matches the unified PR report style (size + perf sections).
*/
const lcovPath = process.argv[2] || 'coverage/playwright/coverage.lcov'
if (!existsSync(lcovPath)) {
process.stdout.write(
'## 🔬 E2E Coverage\n\n> ⚠️ No coverage data found. Check the CI workflow logs.\n'
)
process.exit(0)
}
const lcov = readFileSync(lcovPath, 'utf-8')
// Parse lcov summary
let totalLines = 0
let coveredLines = 0
let totalFunctions = 0
let coveredFunctions = 0
let totalBranches = 0
let coveredBranches = 0
/** @type {Map<string, { lines: number, covered: number }>} */
const fileStats = new Map()
let currentFile = ''
for (const line of lcov.split('\n')) {
if (line.startsWith('SF:')) {
currentFile = line.slice(3)
} else if (line.startsWith('LF:')) {
const n = parseInt(line.slice(3), 10)
totalLines += n
const entry = fileStats.get(currentFile) ?? { lines: 0, covered: 0 }
entry.lines = n
fileStats.set(currentFile, entry)
} else if (line.startsWith('LH:')) {
const n = parseInt(line.slice(3), 10)
coveredLines += n
const entry = fileStats.get(currentFile) ?? { lines: 0, covered: 0 }
entry.covered = n
fileStats.set(currentFile, entry)
} else if (line.startsWith('FNF:')) {
totalFunctions += parseInt(line.slice(4), 10)
} else if (line.startsWith('FNH:')) {
coveredFunctions += parseInt(line.slice(4), 10)
} else if (line.startsWith('BRF:')) {
totalBranches += parseInt(line.slice(4), 10)
} else if (line.startsWith('BRH:')) {
coveredBranches += parseInt(line.slice(4), 10)
}
}
/** @param {number} covered @param {number} total */
function pct(covered, total) {
if (total === 0) return '—'
return ((covered / total) * 100).toFixed(1) + '%'
}
/** @param {number} covered @param {number} total */
function bar(covered, total) {
if (total === 0) return '—'
const p = (covered / total) * 100
if (p >= 80) return '🟢'
if (p >= 50) return '🟡'
return '🔴'
}
const lines = []
lines.push('## 🔬 E2E Coverage')
lines.push('')
lines.push(
'| Metric | Covered | Total | Pct | |'
)
lines.push('|---|--:|--:|--:|---|')
lines.push(
`| Lines | ${coveredLines.toLocaleString()} | ${totalLines.toLocaleString()} | ${pct(coveredLines, totalLines)} | ${bar(coveredLines, totalLines)} |`
)
lines.push(
`| Functions | ${coveredFunctions.toLocaleString()} | ${totalFunctions.toLocaleString()} | ${pct(coveredFunctions, totalFunctions)} | ${bar(coveredFunctions, totalFunctions)} |`
)
lines.push(
`| Branches | ${coveredBranches.toLocaleString()} | ${totalBranches.toLocaleString()} | ${pct(coveredBranches, totalBranches)} | ${bar(coveredBranches, totalBranches)} |`
)
// Top uncovered files
const uncovered = [...fileStats.entries()]
.filter(([, s]) => s.lines > 0)
.map(([file, s]) => ({
file: file.replace(/^.*\/src\//, 'src/'),
pct: s.lines > 0 ? (s.covered / s.lines) * 100 : 100,
missed: s.lines - s.covered
}))
.filter((f) => f.missed > 0)
.sort((a, b) => b.missed - a.missed)
.slice(0, 10)
if (uncovered.length > 0) {
lines.push('')
lines.push('<details>')
lines.push('<summary>Top 10 files by uncovered lines</summary>')
lines.push('')
lines.push('| File | Coverage | Missed |')
lines.push('|---|--:|--:|')
for (const f of uncovered) {
lines.push(`| \`${f.file}\` | ${f.pct.toFixed(1)}% | ${f.missed} |`)
}
lines.push('')
lines.push('</details>')
}
process.stdout.write(lines.join('\n') + '\n')

View File

@@ -13,6 +13,7 @@ function getArg(name) {
const sizeStatus = getArg('size-status') ?? 'pending'
const perfStatus = getArg('perf-status') ?? 'pending'
const coverageStatus = getArg('coverage-status') ?? 'skip'
/** @type {string[]} */
const lines = []
@@ -72,4 +73,30 @@ if (perfStatus === 'ready' && existsSync('test-results/perf-metrics.json')) {
lines.push('> ⏳ Performance tests in progress…')
}
// --- Coverage section ---
if (coverageStatus === 'ready' && existsSync('temp/coverage/coverage.lcov')) {
try {
const coverageReport = execFileSync(
'node',
['scripts/coverage-report.js', 'temp/coverage/coverage.lcov'],
{ encoding: 'utf-8' }
).trimEnd()
lines.push('')
lines.push(coverageReport)
} catch {
lines.push('')
lines.push('## 🔬 E2E Coverage')
lines.push('')
lines.push(
'> ⚠️ Failed to render coverage report. Check the CI workflow logs.'
)
}
} else if (coverageStatus === 'failed') {
lines.push('')
lines.push('## 🔬 E2E Coverage')
lines.push('')
lines.push('> ⚠️ Coverage collection failed. Check the CI workflow logs.')
}
// coverageStatus === 'skip' (default) — don't show section at all
process.stdout.write(lines.join('\n') + '\n')