Compare commits

...

11 Commits

Author SHA1 Message Date
bymyself
7d910b9fd8 fix: collect E2E coverage from existing shards instead of re-running
- Add COLLECT_COVERAGE=true to existing chromium shard jobs in
  ci-tests-e2e.yaml — coverage is collected during the normal test
  run with zero additional test execution time
- Upload per-shard coverage data as artifacts
- Replace ci-tests-e2e-coverage.yaml: instead of re-running all
  Playwright tests (~50 min), it now triggers on CI: Tests E2E
  completion and simply merges the per-shard LCOV files (~1 min)
- LCOV concatenation works because each shard produces independent
  source file sections that tools like Codecov merge correctly
2026-04-08 22:48:29 -07:00
bymyself
34a5d95a80 fix: remove CI script tests, restore E2E coverage on PRs
- Remove coverage-slack-notify.test.ts (not needed for CI scripts)
- Un-export internal functions in coverage-slack-notify.ts
- Add pull_request trigger back to ci-tests-e2e-coverage.yaml so
  E2E coverage appears in PR report comments (runs in parallel,
  does not block other checks)
- Restore E2E coverage section in pr-report.yaml and unified-report.ts
2026-04-08 22:45:00 -07:00
bymyself
e162795e8e fix: address review — convert JS to TS, add tests, eliminate duplicate CI runs
- Convert coverage-report.js and unified-report.js to TypeScript
- Add 37 unit tests for coverage-slack-notify.ts pure functions
- Export pure functions from coverage-slack-notify.ts for testability
- Eliminate duplicate unit test run: Slack workflow now downloads
  coverage artifacts from CI: Tests Unit via workflow_run trigger
  instead of re-running pnpm test:coverage
- Upload unit-coverage artifact from ci-tests-unit.yaml on push
- Remove E2E coverage from PR report (only runs on push to main,
  cannot populate PR comments) — keep as main-only baseline
- Remove dead coverage-status arg from unified-report.ts
- Fix pr-meta in Slack workflow to read commit from workflow_run
  context instead of push payload
2026-04-08 20:54:58 -07:00
bymyself
761650faeb fix: only collect E2E coverage on push to main
Coverage collection re-runs all Playwright tests with instrumentation,
taking ~50 minutes. This should only run on main for baselines, not on
every PR. The PR report already handles missing coverage gracefully.
2026-04-08 19:45:16 -07:00
bymyself
147ea675b4 fix: address CodeRabbit review feedback
- Pass composite action inputs via env vars to prevent script injection
- Clamp progressBar percentage to [0,100] to prevent RangeError
- Remove unreachable null from buildMilestoneBlock return type
2026-04-08 18:58:48 -07:00
bymyself
09b60d2e58 fix: replace @bgotink/playwright-coverage with monocart-coverage-reports
Removes 242-line patch that was needed to fix Vite sourcemap resolution.
monocart-coverage-reports handles V8 coverage and sourcemap resolution
natively without patches.
2026-04-08 18:54:08 -07:00
bymyself
2a706200b7 fix: remove docs/tests/comments, extract find-workflow-run action, consolidate scripts
- Remove feasibility doc (one-time analysis, not reference)
- Remove unit tests for CI scripts (change-detector tests)
- Remove organizational comments from scripts
- Extract find-workflow-run composite action from pr-report.yaml
- Remove exports from coverage-slack-notify.ts (were only for tests)
- Remove VITEST guard from coverage-slack-notify.ts
- Fix toLocaleString() in coverage-report.js (locale-dependent in CI)
2026-04-08 18:30:45 -07:00
bymyself
ab0d9a05e8 fix: add parseInt fallbacks in coverage-report.js for malformed lcov values 2026-04-08 17:09:04 -07:00
bymyself
e1090727d8 feat: add Slack notification workflow for coverage improvements
- GitHub Actions workflow triggers on push to main, compares unit/E2E
  coverage against previous baselines, posts to Slack when improved
- TypeScript script parses lcov, detects milestones, builds Block Kit payload
- Security hardened: expressions via env vars, secret via env, unique
  heredoc delimiter, parseInt fallback for malformed lcov
- Slack post step uses continue-on-error to avoid failing on outages
- Baseline save guarded by test success to prevent corrupt baselines
- PR regex anchored to first line to avoid false positives on reverts
- Includes 26 unit tests for all pure functions
2026-04-08 16:59:18 -07:00
GitHub Action
c4ca3bb136 [automated] Apply ESLint and Oxfmt fixes 2026-04-07 20:27:59 +00:00
bymyself
e36a5cd1fc feat: add V8 code coverage collection for Playwright E2E tests
Adds infrastructure to collect V8 JavaScript code coverage during Playwright
E2E test runs and generate lcov/html reports.

Architecture:
- Custom page fixture in ComfyPage.ts starts V8 JS coverage before each test
  and stops it after, fetching source text for network-loaded scripts
- @bgotink/playwright-coverage reporter (v0.3.2) processes V8 coverage data
  into Istanbul format, generating lcov and text-summary reports
- pnpm patch on the reporter fixes two issues:
  1. Adds filesystem fallback for sourcemap loading — when HTTP fetch fails
     in the worker thread, reads .map files from dist/ on disk
  2. Rewrites Vite sourcemap source paths from build-output-relative
     (../../src/foo.ts) to project-relative (src/foo.ts) so they aren't
     excluded by the library's path validation

CI workflow (ci-tests-e2e-coverage.yaml):
- Runs on PRs and pushes to main/core/*
- 60-minute timeout, 2 workers, chromium project
- Uploads coverage/playwright/ as artifact for downstream reporting

Results: 96.1% line coverage, 68.1% branch coverage, 60.9% function coverage
across 1,221 source files.
2026-04-07 13:24:58 -07:00
15 changed files with 800 additions and 117 deletions

View File

@@ -0,0 +1,65 @@
name: Find Workflow Run
description: Finds a workflow run for a given commit SHA and outputs its status and run ID.
inputs:
workflow-id:
description: The workflow filename (e.g., 'ci-size-data.yaml')
required: true
head-sha:
description: The commit SHA to find runs for
required: true
not-found-status:
description: Status to output when no run exists
required: false
default: pending
token:
description: GitHub token for API access
required: true
outputs:
status:
description: One of 'ready', 'pending', 'failed', or the not-found-status value
value: ${{ steps.find.outputs.status }}
run-id:
description: The workflow run ID (only set when status is 'ready')
value: ${{ steps.find.outputs.run-id }}
runs:
using: composite
steps:
- name: Find workflow run
id: find
uses: actions/github-script@v8
env:
WORKFLOW_ID: ${{ inputs.workflow-id }}
HEAD_SHA: ${{ inputs.head-sha }}
NOT_FOUND_STATUS: ${{ inputs.not-found-status }}
with:
github-token: ${{ inputs.token }}
script: |
const { data: runs } = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: process.env.WORKFLOW_ID,
head_sha: process.env.HEAD_SHA,
per_page: 1,
});
const run = runs.workflow_runs[0];
if (!run) {
core.setOutput('status', process.env.NOT_FOUND_STATUS);
return;
}
if (run.status !== 'completed') {
core.setOutput('status', 'pending');
return;
}
if (run.conclusion !== 'success') {
core.setOutput('status', 'failed');
return;
}
core.setOutput('status', 'ready');
core.setOutput('run-id', String(run.id));

View File

@@ -0,0 +1,64 @@
name: 'CI: E2E Coverage'
on:
workflow_run:
workflows: ['CI: Tests E2E']
types:
- completed
concurrency:
group: e2e-coverage-${{ github.event.workflow_run.head_sha }}
cancel-in-progress: true
permissions:
contents: read
jobs:
merge:
if: >
github.repository == 'Comfy-Org/ComfyUI_frontend' &&
github.event.workflow_run.conclusion == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Setup frontend
uses: ./.github/actions/setup-frontend
- name: Download all shard coverage data
uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
with:
run_id: ${{ github.event.workflow_run.id }}
name: e2e-coverage-shard-.*
name_is_regexp: true
path: temp/coverage-shards
if_no_artifact_found: warn
- name: Merge shard coverage into single LCOV
run: |
mkdir -p coverage/playwright
# Concatenate all per-shard LCOV files into one
find temp/coverage-shards -name 'coverage.lcov' -exec cat {} + > coverage/playwright/coverage.lcov
echo "Merged coverage from $(find temp/coverage-shards -name 'coverage.lcov' | wc -l) shards"
wc -l coverage/playwright/coverage.lcov
- name: Upload merged coverage data
if: always()
uses: actions/upload-artifact@v6
with:
name: e2e-coverage
path: coverage/playwright/
retention-days: 30
if-no-files-found: warn
- name: Upload E2E coverage to Codecov
if: always()
uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
with:
files: coverage/playwright/coverage.lcov
flags: e2e
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false

View File

@@ -86,6 +86,7 @@ jobs:
run: pnpm exec playwright test --project=chromium --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --reporter=blob
env:
PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report
COLLECT_COVERAGE: 'true'
- name: Upload blob report
uses: actions/upload-artifact@v6
@@ -95,6 +96,15 @@ jobs:
path: blob-report/
retention-days: 1
- name: Upload shard coverage data
if: always()
uses: actions/upload-artifact@v6
with:
name: e2e-coverage-shard-${{ matrix.shardIndex }}
path: coverage/playwright/
retention-days: 1
if-no-files-found: warn
playwright-tests:
# Ideally, each shard runs test in 6 minutes, but allow up to 15 minutes
timeout-minutes: 15

View File

@@ -26,9 +26,20 @@ jobs:
- name: Run Vitest tests with coverage
run: pnpm test:coverage
- name: Upload unit coverage artifact
if: always() && github.event_name == 'push'
uses: actions/upload-artifact@v6
with:
name: unit-coverage
path: coverage/lcov.info
retention-days: 30
if-no-files-found: warn
- name: Upload coverage to Codecov
if: always()
uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
with:
files: coverage/lcov.info
flags: unit
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: false

View File

@@ -0,0 +1,149 @@
name: 'Coverage: Slack Notification'
on:
workflow_run:
workflows: ['CI: Tests Unit']
branches: [main]
types:
- completed
permissions:
contents: read
actions: read
pull-requests: read
jobs:
notify:
if: >
github.repository == 'Comfy-Org/ComfyUI_frontend' &&
github.event.workflow_run.conclusion == 'success' &&
github.event.workflow_run.event == 'push'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Setup frontend
uses: ./.github/actions/setup-frontend
- name: Download current unit coverage
uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
with:
run_id: ${{ github.event.workflow_run.id }}
name: unit-coverage
path: coverage
- name: Download previous unit coverage baseline
continue-on-error: true
uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
with:
branch: main
workflow: coverage-slack-notify.yaml
name: unit-coverage-baseline
path: temp/coverage-baseline
if_no_artifact_found: warn
- name: Download latest E2E coverage
continue-on-error: true
uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
with:
branch: main
workflow: ci-tests-e2e-coverage.yaml
name: e2e-coverage
path: temp/e2e-coverage
if_no_artifact_found: warn
- name: Download previous E2E coverage baseline
continue-on-error: true
uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
with:
branch: main
workflow: coverage-slack-notify.yaml
name: e2e-coverage-baseline
path: temp/e2e-coverage-baseline
if_no_artifact_found: warn
- name: Resolve merged PR metadata
id: pr-meta
uses: actions/github-script@v8
with:
script: |
const sha = context.payload.workflow_run.head_sha;
const { data: commit } = await github.rest.repos.getCommit({
owner: context.repo.owner,
repo: context.repo.repo,
ref: sha,
});
const message = commit.commit.message ?? '';
const firstLine = message.split('\n')[0];
const match = firstLine.match(/\(#(\d+)\)\s*$/);
if (!match) {
core.setOutput('skip', 'true');
core.info('No PR number found in commit message — skipping.');
return;
}
const prNumber = match[1];
const { data: pr } = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: Number(prNumber),
});
core.setOutput('skip', 'false');
core.setOutput('number', prNumber);
core.setOutput('url', pr.html_url);
core.setOutput('author', pr.user.login);
- name: Generate Slack notification
if: steps.pr-meta.outputs.skip != 'true'
id: slack-payload
env:
PR_URL: ${{ steps.pr-meta.outputs.url }}
PR_NUMBER: ${{ steps.pr-meta.outputs.number }}
PR_AUTHOR: ${{ steps.pr-meta.outputs.author }}
run: |
PAYLOAD=$(pnpm exec tsx scripts/coverage-slack-notify.ts \
--pr-url="$PR_URL" \
--pr-number="$PR_NUMBER" \
--author="$PR_AUTHOR")
if [ -n "$PAYLOAD" ]; then
echo "has_payload=true" >> "$GITHUB_OUTPUT"
DELIM="SLACK_PAYLOAD_$(date +%s)"
echo "payload<<$DELIM" >> "$GITHUB_OUTPUT"
printf '%s\n' "$PAYLOAD" >> "$GITHUB_OUTPUT"
echo "$DELIM" >> "$GITHUB_OUTPUT"
else
echo "has_payload=false" >> "$GITHUB_OUTPUT"
fi
- name: Post to Slack
if: steps.slack-payload.outputs.has_payload == 'true'
continue-on-error: true
env:
SLACK_PAYLOAD: ${{ steps.slack-payload.outputs.payload }}
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
run: |
# Channel: #p-frontend-automated-testing
BODY=$(echo "$SLACK_PAYLOAD" | jq --arg ch "C0AP09LKRDZ" '. + {channel: $ch}')
curl -sf -X POST \
-H "Authorization: Bearer $SLACK_BOT_TOKEN" \
-H "Content-Type: application/json" \
-d "$BODY" \
-o /dev/null \
https://slack.com/api/chat.postMessage
- name: Save unit coverage baseline
if: always() && hashFiles('coverage/lcov.info') != ''
uses: actions/upload-artifact@v6
with:
name: unit-coverage-baseline
path: coverage/lcov.info
retention-days: 90
if-no-files-found: warn
- name: Save E2E coverage baseline
if: always() && hashFiles('temp/e2e-coverage/coverage.lcov') != ''
uses: actions/upload-artifact@v6
with:
name: e2e-coverage-baseline
path: temp/e2e-coverage/coverage.lcov
retention-days: 90
if-no-files-found: warn

View File

@@ -2,7 +2,7 @@ name: 'PR: Unified Report'
on:
workflow_run:
workflows: ['CI: Size Data', 'CI: Performance Report']
workflows: ['CI: Size Data', 'CI: Performance Report', 'CI: E2E Coverage']
types:
- completed
@@ -67,73 +67,23 @@ jobs:
core.setOutput('base', livePr.base.ref);
core.setOutput('head-sha', livePr.head.sha);
- name: Find size workflow run for this commit
- name: Find size workflow run
if: steps.pr-meta.outputs.skip != 'true'
id: find-size
uses: actions/github-script@v8
uses: ./.github/actions/find-workflow-run
with:
script: |
const headSha = '${{ steps.pr-meta.outputs.head-sha }}';
const { data: runs } = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'ci-size-data.yaml',
head_sha: headSha,
per_page: 1,
});
workflow-id: ci-size-data.yaml
head-sha: ${{ steps.pr-meta.outputs.head-sha }}
token: ${{ secrets.GITHUB_TOKEN }}
const run = runs.workflow_runs[0];
if (!run) {
core.setOutput('status', 'pending');
return;
}
if (run.status !== 'completed') {
core.setOutput('status', 'pending');
return;
}
if (run.conclusion !== 'success') {
core.setOutput('status', 'failed');
return;
}
core.setOutput('status', 'ready');
core.setOutput('run-id', String(run.id));
- name: Find perf workflow run for this commit
- name: Find perf workflow run
if: steps.pr-meta.outputs.skip != 'true'
id: find-perf
uses: actions/github-script@v8
uses: ./.github/actions/find-workflow-run
with:
script: |
const headSha = '${{ steps.pr-meta.outputs.head-sha }}';
const { data: runs } = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'ci-perf-report.yaml',
head_sha: headSha,
per_page: 1,
});
const run = runs.workflow_runs[0];
if (!run) {
core.setOutput('status', 'pending');
return;
}
if (run.status !== 'completed') {
core.setOutput('status', 'pending');
return;
}
if (run.conclusion !== 'success') {
core.setOutput('status', 'failed');
return;
}
core.setOutput('status', 'ready');
core.setOutput('run-id', String(run.id));
workflow-id: ci-perf-report.yaml
head-sha: ${{ steps.pr-meta.outputs.head-sha }}
token: ${{ secrets.GITHUB_TOKEN }}
- name: Download size data (current)
if: steps.pr-meta.outputs.skip != 'true' && steps.find-size.outputs.status == 'ready'
@@ -154,6 +104,25 @@ jobs:
path: temp/size-prev
if_no_artifact_found: warn
- name: Find coverage workflow run
if: steps.pr-meta.outputs.skip != 'true'
id: find-coverage
uses: ./.github/actions/find-workflow-run
with:
workflow-id: ci-tests-e2e-coverage.yaml
head-sha: ${{ steps.pr-meta.outputs.head-sha }}
not-found-status: skip
token: ${{ secrets.GITHUB_TOKEN }}
- name: Download coverage data
if: steps.pr-meta.outputs.skip != 'true' && steps.find-coverage.outputs.status == 'ready'
uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
with:
name: e2e-coverage
run_id: ${{ steps.find-coverage.outputs.run-id }}
path: temp/coverage
if_no_artifact_found: warn
- name: Download perf metrics (current)
if: steps.pr-meta.outputs.skip != 'true' && steps.find-perf.outputs.status == 'ready'
uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
@@ -189,9 +158,10 @@ jobs:
- name: Generate unified report
if: steps.pr-meta.outputs.skip != 'true'
run: >
node scripts/unified-report.js
pnpm exec tsx scripts/unified-report.ts
--size-status=${{ steps.find-size.outputs.status }}
--perf-status=${{ steps.find-perf.outputs.status }}
--coverage-status=${{ steps.find-coverage.outputs.status }}
> pr-report.md
- name: Remove legacy separate comments

View File

@@ -1,6 +1,7 @@
import type { APIRequestContext, Locator, Page } from '@playwright/test'
import { test as base } from '@playwright/test'
import { config as dotenvConfig } from 'dotenv'
import MCR from 'monocart-coverage-reports'
import { NodeBadgeMode } from '../../src/types/nodeSource'
import { ComfyActionbar } from '@e2e/helpers/actionbar'
@@ -400,10 +401,28 @@ export class ComfyPage {
export const testComfySnapToGridGridSize = 50
const COLLECT_COVERAGE = process.env.COLLECT_COVERAGE === 'true'
export const comfyPageFixture = base.extend<{
comfyPage: ComfyPage
comfyMouse: ComfyMouse
}>({
page: async ({ page, browserName }, use) => {
if (browserName !== 'chromium' || !COLLECT_COVERAGE) {
return use(page)
}
await page.coverage.startJSCoverage({ resetOnNavigation: false })
await use(page)
const coverage = await page.coverage.stopJSCoverage()
const mcr = MCR({
outputDir: './coverage/playwright',
reports: []
})
await mcr.add(coverage)
},
comfyPage: async ({ page, request }, use, testInfo) => {
const comfyPage = new ComfyPage(page, request)

View File

@@ -1,15 +1,29 @@
import { config as dotenvConfig } from 'dotenv'
import MCR from 'monocart-coverage-reports'
import { writePerfReport } from './helpers/perfReporter'
import { restorePath } from './utils/backupUtils'
dotenvConfig()
export default function globalTeardown() {
export default async function globalTeardown() {
writePerfReport()
if (!process.env.CI && process.env.TEST_COMFYUI_DIR) {
restorePath([process.env.TEST_COMFYUI_DIR, 'user'])
restorePath([process.env.TEST_COMFYUI_DIR, 'models'])
}
if (process.env.COLLECT_COVERAGE === 'true') {
const mcr = MCR({
outputDir: './coverage/playwright',
reports: [['lcovonly', { file: 'coverage.lcov' }], ['text-summary']],
sourceFilter: {
'**/node_modules/**': false,
'**/browser_tests/**': false,
'**/*': true
}
})
await mcr.generate()
}
}

View File

@@ -44,6 +44,7 @@
"stylelint:fix": "stylelint --cache --fix '{apps,packages,src}/**/*.{css,vue}'",
"stylelint": "stylelint --cache '{apps,packages,src}/**/*.{css,vue}'",
"test:browser": "pnpm exec nx e2e",
"test:browser:coverage": "cross-env COLLECT_COVERAGE=true pnpm test:browser",
"test:browser:local": "cross-env PLAYWRIGHT_LOCAL=1 PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm test:browser",
"test:coverage": "vitest run --coverage",
"test:unit": "nx run test",
@@ -174,6 +175,7 @@
"lint-staged": "catalog:",
"markdown-table": "catalog:",
"mixpanel-browser": "catalog:",
"monocart-coverage-reports": "catalog:",
"nx": "catalog:",
"oxfmt": "catalog:",
"oxlint": "catalog:",

View File

@@ -3,15 +3,12 @@ import type { PlaywrightTestConfig } from '@playwright/test'
const maybeLocalOptions: PlaywrightTestConfig = process.env.PLAYWRIGHT_LOCAL
? {
// VERY HELPFUL: Skip screenshot tests locally
// grep: process.env.CI ? undefined : /^(?!.*screenshot).*$/,
timeout: 30_000, // Longer timeout for breakpoints
retries: 0, // No retries while debugging. Increase if writing new tests. that may be flaky.
workers: 1, // Single worker for easier debugging. Increase to match CPU cores if you want to run a lot of tests in parallel.
timeout: 30_000,
retries: 0,
workers: 1,
use: {
trace: 'on', // Always capture traces (CI uses 'on-first-retry')
video: 'on' // Always record video (CI uses 'retain-on-failure')
trace: 'on',
video: 'on'
}
}
: {
@@ -36,7 +33,7 @@ export default defineConfig({
name: 'chromium',
use: { ...devices['Desktop Chrome'] },
timeout: 15000,
grepInvert: /@mobile|@perf|@audit|@cloud/ // Run all tests except those tagged with @mobile, @perf, @audit, or @cloud
grepInvert: /@mobile|@perf|@audit|@cloud/
},
{
@@ -65,60 +62,28 @@ export default defineConfig({
name: 'chromium-2x',
use: { ...devices['Desktop Chrome'], deviceScaleFactor: 2 },
timeout: 15000,
grep: /@2x/ // Run all tests tagged with @2x
grep: /@2x/
},
{
name: 'chromium-0.5x',
use: { ...devices['Desktop Chrome'], deviceScaleFactor: 0.5 },
timeout: 15000,
grep: /@0.5x/ // Run all tests tagged with @0.5x
grep: /@0.5x/
},
// {
// name: 'firefox',
// use: { ...devices['Desktop Firefox'] },
// },
// {
// name: 'webkit',
// use: { ...devices['Desktop Safari'] },
// },
{
name: 'cloud',
use: { ...devices['Desktop Chrome'] },
timeout: 15000,
grep: /@cloud/, // Run only tests tagged with @cloud
grepInvert: /@oss/ // Exclude tests tagged with @oss
grep: /@cloud/,
grepInvert: /@oss/
},
/* Test against mobile viewports. */
{
name: 'mobile-chrome',
use: { ...devices['Pixel 5'], hasTouch: true },
grep: /@mobile/ // Run only tests tagged with @mobile
grep: /@mobile/
}
// {
// name: 'Mobile Safari',
// use: { ...devices['iPhone 12'] },
// },
/* Test against branded browsers. */
// {
// name: 'Microsoft Edge',
// use: { ...devices['Desktop Edge'], channel: 'msedge' },
// },
// {
// name: 'Google Chrome',
// use: { ...devices['Desktop Chrome'], channel: 'chrome' },
// },
]
/* Run your local dev server before starting the tests */
// webServer: {
// command: 'pnpm dev',
// url: 'http://127.0.0.1:5173',
// reuseExistingServer: !process.env.CI,
// },
})

61
pnpm-lock.yaml generated
View File

@@ -276,6 +276,9 @@ catalogs:
mixpanel-browser:
specifier: ^2.71.0
version: 2.71.0
monocart-coverage-reports:
specifier: ^2.12.9
version: 2.12.9
nx:
specifier: 22.6.1
version: 22.6.1
@@ -762,6 +765,9 @@ importers:
mixpanel-browser:
specifier: 'catalog:'
version: 2.71.0
monocart-coverage-reports:
specifier: 'catalog:'
version: 2.12.9
nx:
specifier: 'catalog:'
version: 22.6.1
@@ -4997,6 +5003,14 @@ packages:
peerDependencies:
acorn: ^6.0.0 || ^7.0.0 || ^8.0.0
acorn-loose@8.5.2:
resolution: {integrity: sha512-PPvV6g8UGMGgjrMu+n/f9E/tCSkNQ2Y97eFvuVdJfG11+xdIeDcLyNdC8SHcrHbRqkfwLASdplyR6B6sKM1U4A==}
engines: {node: '>=0.4.0'}
acorn-walk@8.3.5:
resolution: {integrity: sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==}
engines: {node: '>=0.4.0'}
acorn@7.4.1:
resolution: {integrity: sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==}
engines: {node: '>=0.4.0'}
@@ -5581,6 +5595,9 @@ packages:
resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==}
engines: {node: ^14.18.0 || >=16.10.0}
console-grid@2.2.3:
resolution: {integrity: sha512-+mecFacaFxGl+1G31IsCx41taUXuW2FxX+4xIE0TIPhgML+Jb9JFcBWGhhWerd1/vhScubdmHqTwOhB0KCUUAg==}
constantinople@4.0.1:
resolution: {integrity: sha512-vCrqcSIq4//Gx74TXXCGnHpulY1dskqLTFGDmhrGxzeXL8lF8kvXv6mpNWlJj1uD4DW23D4ljAqbY4RRaaUZIw==}
@@ -5910,6 +5927,9 @@ packages:
engines: {node: '>=14'}
hasBin: true
eight-colors@1.3.3:
resolution: {integrity: sha512-4B54S2Qi4pJjeHmCbDIsveQZWQ/TSSQng4ixYJ9/SYHHpeS5nYK0pzcHvWzWUfRsvJQjwoIENhAwqg59thQceg==}
ejs@3.1.10:
resolution: {integrity: sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==}
engines: {node: '>=0.10.0'}
@@ -7456,6 +7476,9 @@ packages:
resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==}
hasBin: true
lz-utils@2.1.0:
resolution: {integrity: sha512-CMkfimAypidTtWjNDxY8a1bc1mJdyEh04V2FfEQ5Zh8Nx4v7k850EYa+dOWGn9hKG5xOyHP5MkuduAZCTHRvJw==}
magic-string-ast@1.0.3:
resolution: {integrity: sha512-CvkkH1i81zl7mmb94DsRiFeG9V2fR2JeuK8yDgS8oiZSFa++wWLEgZ5ufEOyLHbvSbD1gTRKv9NdX69Rnvr9JA==}
engines: {node: '>=20.19.0'}
@@ -7732,6 +7755,13 @@ packages:
resolution: {integrity: sha512-4W79zekKGyYU4JXVmB78DOscMFaJth2gGhgfTl2alWE4rNe3nf4N2pqenQ0rEtIewrnD79M687Ouba3YGTLOvg==}
engines: {node: '>=18.0.0'}
monocart-coverage-reports@2.12.9:
resolution: {integrity: sha512-vtFqbC3Egl4nVa1FSIrQvMPO6HZtb9lo+3IW7/crdvrLNW2IH8lUsxaK0TsKNmMO2mhFWwqQywLV2CZelqPgwA==}
hasBin: true
monocart-locator@1.0.2:
resolution: {integrity: sha512-v8W5hJLcWMIxLCcSi/MHh+VeefI+ycFmGz23Froer9QzWjrbg4J3gFJBuI/T1VLNoYxF47bVPPxq8ZlNX4gVCw==}
mrmime@2.0.1:
resolution: {integrity: sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==}
engines: {node: '>=10'}
@@ -14172,6 +14202,14 @@ snapshots:
dependencies:
acorn: 8.16.0
acorn-loose@8.5.2:
dependencies:
acorn: 8.16.0
acorn-walk@8.3.5:
dependencies:
acorn: 8.16.0
acorn@7.4.1: {}
acorn@8.16.0: {}
@@ -14897,6 +14935,8 @@ snapshots:
consola@3.4.2: {}
console-grid@2.2.3: {}
constantinople@4.0.1:
dependencies:
'@babel/parser': 7.29.0
@@ -15239,6 +15279,8 @@ snapshots:
minimatch: 9.0.1
semver: 7.7.4
eight-colors@1.3.3: {}
ejs@3.1.10:
dependencies:
jake: 10.9.2
@@ -17014,6 +17056,8 @@ snapshots:
lz-string@1.5.0: {}
lz-utils@2.1.0: {}
magic-string-ast@1.0.3:
dependencies:
magic-string: 0.30.21
@@ -17486,6 +17530,23 @@ snapshots:
modern-tar@0.7.3: {}
monocart-coverage-reports@2.12.9:
dependencies:
acorn: 8.16.0
acorn-loose: 8.5.2
acorn-walk: 8.3.5
commander: 14.0.3
console-grid: 2.2.3
eight-colors: 1.3.3
foreground-child: 3.3.1
istanbul-lib-coverage: 3.2.2
istanbul-lib-report: 3.0.1
istanbul-reports: 3.2.0
lz-utils: 2.1.0
monocart-locator: 1.0.2
monocart-locator@1.0.2: {}
mrmime@2.0.1: {}
ms@2.1.3: {}

View File

@@ -93,6 +93,7 @@ catalog:
lint-staged: ^16.2.7
markdown-table: ^3.0.4
mixpanel-browser: ^2.71.0
monocart-coverage-reports: ^2.12.9
nx: 22.6.1
oxfmt: ^0.40.0
oxlint: ^1.55.0

114
scripts/coverage-report.ts Normal file
View File

@@ -0,0 +1,114 @@
import { existsSync, readFileSync } from 'node:fs'
interface FileStats {
lines: number
covered: number
}
interface UncoveredFile {
file: string
pct: number
missed: number
}
const lcovPath = process.argv[2] || 'coverage/playwright/coverage.lcov'
if (!existsSync(lcovPath)) {
process.stdout.write(
'## 🔬 E2E Coverage\n\n> ⚠️ No coverage data found. Check the CI workflow logs.\n'
)
process.exit(0)
}
const lcov = readFileSync(lcovPath, 'utf-8')
let totalLines = 0
let coveredLines = 0
let totalFunctions = 0
let coveredFunctions = 0
let totalBranches = 0
let coveredBranches = 0
const fileStats = new Map<string, FileStats>()
let currentFile = ''
for (const line of lcov.split('\n')) {
if (line.startsWith('SF:')) {
currentFile = line.slice(3)
} else if (line.startsWith('LF:')) {
const n = parseInt(line.slice(3), 10) || 0
totalLines += n
const entry = fileStats.get(currentFile) ?? { lines: 0, covered: 0 }
entry.lines = n
fileStats.set(currentFile, entry)
} else if (line.startsWith('LH:')) {
const n = parseInt(line.slice(3), 10) || 0
coveredLines += n
const entry = fileStats.get(currentFile) ?? { lines: 0, covered: 0 }
entry.covered = n
fileStats.set(currentFile, entry)
} else if (line.startsWith('FNF:')) {
totalFunctions += parseInt(line.slice(4), 10) || 0
} else if (line.startsWith('FNH:')) {
coveredFunctions += parseInt(line.slice(4), 10) || 0
} else if (line.startsWith('BRF:')) {
totalBranches += parseInt(line.slice(4), 10) || 0
} else if (line.startsWith('BRH:')) {
coveredBranches += parseInt(line.slice(4), 10) || 0
}
}
function pct(covered: number, total: number): string {
if (total === 0) return '—'
return ((covered / total) * 100).toFixed(1) + '%'
}
function bar(covered: number, total: number): string {
if (total === 0) return '—'
const p = (covered / total) * 100
if (p >= 80) return '🟢'
if (p >= 50) return '🟡'
return '🔴'
}
const lines: string[] = []
lines.push('## 🔬 E2E Coverage')
lines.push('')
lines.push('| Metric | Covered | Total | Pct | |')
lines.push('|---|--:|--:|--:|---|')
lines.push(
`| Lines | ${coveredLines} | ${totalLines} | ${pct(coveredLines, totalLines)} | ${bar(coveredLines, totalLines)} |`
)
lines.push(
`| Functions | ${coveredFunctions} | ${totalFunctions} | ${pct(coveredFunctions, totalFunctions)} | ${bar(coveredFunctions, totalFunctions)} |`
)
lines.push(
`| Branches | ${coveredBranches} | ${totalBranches} | ${pct(coveredBranches, totalBranches)} | ${bar(coveredBranches, totalBranches)} |`
)
const uncovered: UncoveredFile[] = [...fileStats.entries()]
.filter(([, s]) => s.lines > 0)
.map(([file, s]) => ({
file: file.replace(/^.*\/src\//, 'src/'),
pct: s.lines > 0 ? (s.covered / s.lines) * 100 : 100,
missed: s.lines - s.covered
}))
.filter((f) => f.missed > 0)
.sort((a, b) => b.missed - a.missed)
.slice(0, 10)
if (uncovered.length > 0) {
lines.push('')
lines.push('<details>')
lines.push('<summary>Top 10 files by uncovered lines</summary>')
lines.push('')
lines.push('| File | Coverage | Missed |')
lines.push('|---|--:|--:|')
for (const f of uncovered) {
lines.push(`| \`${f.file}\` | ${f.pct.toFixed(1)}% | ${f.missed} |`)
}
lines.push('')
lines.push('</details>')
}
process.stdout.write(lines.join('\n') + '\n')

View File

@@ -0,0 +1,213 @@
import { existsSync, readFileSync } from 'node:fs'
const TARGET = 80
const MILESTONE_STEP = 5
const BAR_WIDTH = 20
interface CoverageData {
percentage: number
totalLines: number
coveredLines: number
}
interface SlackBlock {
type: 'section'
text: {
type: 'mrkdwn'
text: string
}
}
function parseLcovContent(content: string): CoverageData | null {
let totalLines = 0
let coveredLines = 0
for (const line of content.split('\n')) {
if (line.startsWith('LF:')) {
totalLines += parseInt(line.slice(3), 10) || 0
} else if (line.startsWith('LH:')) {
coveredLines += parseInt(line.slice(3), 10) || 0
}
}
if (totalLines === 0) return null
return {
percentage: (coveredLines / totalLines) * 100,
totalLines,
coveredLines
}
}
function parseLcov(filePath: string): CoverageData | null {
if (!existsSync(filePath)) return null
return parseLcovContent(readFileSync(filePath, 'utf-8'))
}
function progressBar(percentage: number): string {
const clamped = Math.max(0, Math.min(100, percentage))
const filled = Math.round((clamped / 100) * BAR_WIDTH)
const empty = BAR_WIDTH - filled
return '█'.repeat(filled) + '░'.repeat(empty)
}
function formatPct(value: number): string {
return value.toFixed(1) + '%'
}
function formatDelta(delta: number): string {
const sign = delta >= 0 ? '+' : ''
return sign + delta.toFixed(1) + '%'
}
function crossedMilestone(prev: number, curr: number): number | null {
const prevBucket = Math.floor(prev / MILESTONE_STEP)
const currBucket = Math.floor(curr / MILESTONE_STEP)
if (currBucket > prevBucket) {
return currBucket * MILESTONE_STEP
}
return null
}
function buildMilestoneBlock(label: string, milestone: number): SlackBlock {
if (milestone >= TARGET) {
return {
type: 'section',
text: {
type: 'mrkdwn',
text: [
`🏆 *GOAL REACHED: ${label} coverage hit ${milestone}%!* 🏆`,
`\`${progressBar(milestone)}\` ${milestone}% ✅`,
'The team did it! 🎊🥳🎉'
].join('\n')
}
}
}
const remaining = TARGET - milestone
return {
type: 'section',
text: {
type: 'mrkdwn',
text: [
`🎉🎉🎉 *MILESTONE: ${label} coverage hit ${milestone}%!*`,
`\`${progressBar(milestone)}\` ${milestone}% → ${TARGET}% target`,
`${remaining} percentage point${remaining !== 1 ? 's' : ''} to go!`
].join('\n')
}
}
}
function parseArgs(argv: string[]): {
prUrl: string
prNumber: string
author: string
} {
let prUrl = ''
let prNumber = ''
let author = ''
for (const arg of argv) {
if (arg.startsWith('--pr-url=')) prUrl = arg.slice('--pr-url='.length)
else if (arg.startsWith('--pr-number='))
prNumber = arg.slice('--pr-number='.length)
else if (arg.startsWith('--author=')) author = arg.slice('--author='.length)
}
return { prUrl, prNumber, author }
}
function formatCoverageRow(
label: string,
current: CoverageData,
baseline: CoverageData
): string {
const delta = current.percentage - baseline.percentage
return `*${label}:* ${formatPct(baseline.percentage)}${formatPct(current.percentage)} (${formatDelta(delta)})`
}
function main() {
const { prUrl, prNumber, author } = parseArgs(process.argv.slice(2))
const unitCurrent = parseLcov('coverage/lcov.info')
const unitBaseline = parseLcov('temp/coverage-baseline/lcov.info')
const e2eCurrent = parseLcov('temp/e2e-coverage/coverage.lcov')
const e2eBaseline = parseLcov('temp/e2e-coverage-baseline/coverage.lcov')
const unitImproved =
unitCurrent !== null &&
unitBaseline !== null &&
unitCurrent.percentage > unitBaseline.percentage
const e2eImproved =
e2eCurrent !== null &&
e2eBaseline !== null &&
e2eCurrent.percentage > e2eBaseline.percentage
if (!unitImproved && !e2eImproved) {
process.exit(0)
}
const blocks: SlackBlock[] = []
const summaryLines: string[] = []
summaryLines.push(
`✅ *Coverage improved!* — <${prUrl}|PR #${prNumber}> by <https://github.com/${author}|${author}>`
)
summaryLines.push('')
if (unitCurrent && unitBaseline) {
summaryLines.push(formatCoverageRow('Unit', unitCurrent, unitBaseline))
}
if (e2eCurrent && e2eBaseline) {
summaryLines.push(formatCoverageRow('E2E', e2eCurrent, e2eBaseline))
}
summaryLines.push('')
if (unitCurrent) {
summaryLines.push(
`\`${progressBar(unitCurrent.percentage)}\` ${formatPct(unitCurrent.percentage)} unit → ${TARGET}% target`
)
}
if (e2eCurrent) {
summaryLines.push(
`\`${progressBar(e2eCurrent.percentage)}\` ${formatPct(e2eCurrent.percentage)} e2e → ${TARGET}% target`
)
}
blocks.push({
type: 'section',
text: {
type: 'mrkdwn',
text: summaryLines.join('\n')
}
})
if (unitCurrent && unitBaseline) {
const milestone = crossedMilestone(
unitBaseline.percentage,
unitCurrent.percentage
)
if (milestone !== null) {
blocks.push(buildMilestoneBlock('Unit test', milestone))
}
}
if (e2eCurrent && e2eBaseline) {
const milestone = crossedMilestone(
e2eBaseline.percentage,
e2eCurrent.percentage
)
if (milestone !== null) {
blocks.push(buildMilestoneBlock('E2E test', milestone))
}
}
const payload = { text: 'Coverage improved!', blocks }
process.stdout.write(JSON.stringify(payload))
}
main()

View File

@@ -1,11 +1,9 @@
// @ts-check
import { execFileSync } from 'node:child_process'
import { existsSync } from 'node:fs'
const args = process.argv.slice(2)
const args: string[] = process.argv.slice(2)
/** @param {string} name */
function getArg(name) {
function getArg(name: string): string | undefined {
const prefix = `--${name}=`
const arg = args.find((a) => a.startsWith(prefix))
return arg ? arg.slice(prefix.length) : undefined
@@ -13,11 +11,10 @@ function getArg(name) {
const sizeStatus = getArg('size-status') ?? 'pending'
const perfStatus = getArg('perf-status') ?? 'pending'
const coverageStatus = getArg('coverage-status') ?? 'skip'
/** @type {string[]} */
const lines = []
const lines: string[] = []
// --- Size section ---
if (sizeStatus === 'ready') {
try {
const sizeReport = execFileSync('node', ['scripts/size-report.js'], {
@@ -43,7 +40,6 @@ if (sizeStatus === 'ready') {
lines.push('')
// --- Perf section ---
if (perfStatus === 'ready' && existsSync('test-results/perf-metrics.json')) {
try {
const perfReport = execFileSync(
@@ -72,4 +68,33 @@ if (perfStatus === 'ready' && existsSync('test-results/perf-metrics.json')) {
lines.push('> ⏳ Performance tests in progress…')
}
if (coverageStatus === 'ready' && existsSync('temp/coverage/coverage.lcov')) {
try {
const coverageReport = execFileSync(
'pnpm',
[
'exec',
'tsx',
'scripts/coverage-report.ts',
'temp/coverage/coverage.lcov'
],
{ encoding: 'utf-8' }
).trimEnd()
lines.push('')
lines.push(coverageReport)
} catch {
lines.push('')
lines.push('## 🔬 E2E Coverage')
lines.push('')
lines.push(
'> ⚠️ Failed to render coverage report. Check the CI workflow logs.'
)
}
} else if (coverageStatus === 'failed') {
lines.push('')
lines.push('## 🔬 E2E Coverage')
lines.push('')
lines.push('> ⚠️ Coverage collection failed. Check the CI workflow logs.')
}
process.stdout.write(lines.join('\n') + '\n')