ci: require exact critical coverage baseline

.github/actions/find-workflow-run/action.yaml

@@ -8,6 +8,10 @@ inputs:
   head-sha:
     description: The commit SHA to find runs for
     required: true
+  event:
+    description: Optional workflow event to match
+    required: false
+    default: ''
   not-found-status:
     description: Status to output when no run exists
     required: false
@@ -33,6 +37,7 @@ runs:
       env:
         WORKFLOW_ID: ${{ inputs.workflow-id }}
         HEAD_SHA: ${{ inputs.head-sha }}
+        EVENT_NAME: ${{ inputs.event }}
         NOT_FOUND_STATUS: ${{ inputs.not-found-status }}
       with:
         github-token: ${{ inputs.token }}
@@ -42,6 +47,7 @@ runs:
             repo: context.repo.repo,
             workflow_id: process.env.WORKFLOW_ID,
             head_sha: process.env.HEAD_SHA,
+            event: process.env.EVENT_NAME || undefined,
             per_page: 1,
           });
 

.github/workflows/ci-tests-unit.yaml

@@ -38,6 +38,18 @@ jobs:
       - name: Run Vitest tests with coverage
         run: pnpm test:coverage
 
+      - name: Generate critical unit coverage artifact
+        run: pnpm coverage:critical:extract
+
+      - name: Upload critical unit coverage artifact
+        if: github.event_name != 'merge_group'
+        uses: actions/upload-artifact@v6
+        with:
+          name: critical-unit-coverage-${{ github.sha }}
+          path: coverage/critical-unit-coverage.json
+          retention-days: 30
+          if-no-files-found: error
+
       - name: Upload unit coverage artifact
         if: always() && github.event_name == 'push'
         uses: actions/upload-artifact@v6
@@ -55,3 +67,79 @@ jobs:
           flags: unit
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: false
+
+  critical-unit-regression:
+    needs: test
+    if: ${{ github.event_name == 'pull_request' && needs.test.result == 'success' }}
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup frontend
+        uses: ./.github/actions/setup-frontend
+
+      - name: Download head critical unit coverage
+        uses: actions/download-artifact@v7
+        with:
+          name: critical-unit-coverage-${{ github.sha }}
+          path: temp/head-critical-coverage
+
+      - name: Find base unit coverage run
+        id: find-base-unit
+        uses: ./.github/actions/find-workflow-run
+        with:
+          workflow-id: ci-tests-unit.yaml
+          head-sha: ${{ github.event.pull_request.base.sha }}
+          event: push
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Require base unit coverage run
+        if: steps.find-base-unit.outputs.status != 'ready'
+        run: |
+          echo "Critical unit baseline artifact unavailable for base ${{ github.event.pull_request.base.sha }}." >> "$GITHUB_STEP_SUMMARY"
+          echo "Status: ${{ steps.find-base-unit.outputs.status }}" >> "$GITHUB_STEP_SUMMARY"
+          exit 1
+
+      - name: Download base critical unit coverage
+        continue-on-error: true
+        uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
+        with:
+          name: critical-unit-coverage-${{ github.event.pull_request.base.sha }}
+          run_id: ${{ steps.find-base-unit.outputs.run-id }}
+          path: temp/base-critical-coverage
+          if_no_artifact_found: warn
+
+      - name: Download base unit LCOV fallback
+        continue-on-error: true
+        uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
+        with:
+          name: unit-coverage
+          run_id: ${{ steps.find-base-unit.outputs.run-id }}
+          path: temp/base-unit-coverage
+          if_no_artifact_found: warn
+
+      - name: Prepare base critical unit coverage fallback
+        run: |
+          if [ -f temp/base-critical-coverage/critical-unit-coverage.json ]; then
+            exit 0
+          fi
+
+          if [ ! -f temp/base-unit-coverage/lcov.info ]; then
+            echo "Critical unit baseline artifact missing for base ${{ github.event.pull_request.base.sha }}." >> "$GITHUB_STEP_SUMMARY"
+            exit 1
+          fi
+
+          pnpm coverage:critical:extract \
+            --input temp/base-unit-coverage/lcov.info \
+            --output temp/base-critical-coverage/critical-unit-coverage.json \
+            --sha=${{ github.event.pull_request.base.sha }}
+
+      - name: Compare critical unit coverage
+        run: >
+          pnpm coverage:critical:compare
+          --base temp/base-critical-coverage/critical-unit-coverage.json
+          --head temp/head-critical-coverage/critical-unit-coverage.json

package.json

@@ -19,6 +19,8 @@
     "size:collect": "node scripts/size-collect.js",
     "size:report": "node scripts/size-report.js",
     "collect-i18n": "pnpm exec playwright test --config=playwright.i18n.config.ts",
+    "coverage:critical:compare": "tsx scripts/critical-coverage/compareCriticalCoverage.ts",
+    "coverage:critical:extract": "tsx scripts/critical-coverage/extractCriticalCoverage.ts",
     "dev:cloud": "pnpm dev:cloud:test",
     "dev:cloud:test": "cross-env DEV_SERVER_COMFYUI_URL=https://testcloud.comfy.org/ vite --config vite.config.mts",
     "dev:cloud:staging": "cross-env DEV_SERVER_COMFYUI_URL=https://stagingcloud.comfy.org/ vite --config vite.config.mts",

scripts/critical-coverage/compareCriticalCoverage.ts

@@ -0,0 +1,120 @@
+import { appendFileSync } from 'node:fs'
+
+import {
+  compareCriticalCoverageReports,
+  readCriticalCoverageReport
+} from './criticalCoverageReport'
+import type { CriticalCoverageComparison } from './criticalCoverageReport'
+
+interface Options {
+  base: string
+  head: string
+}
+
+const options = parseOptions(process.argv.slice(2))
+const base = readCriticalCoverageReport(options.base)
+const head = readCriticalCoverageReport(options.head)
+const comparison = compareCriticalCoverageReports(base, head)
+const summary = formatComparison(comparison)
+
+process.stdout.write(`${summary}\n`)
+
+if (process.env.GITHUB_STEP_SUMMARY) {
+  appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`)
+}
+
+if (comparison.commonBranches === 0) {
+  process.stderr.write('No comparable critical unit branches found.\n')
+  process.exit(1)
+}
+
+if (comparison.coveredBranchDelta < 0) {
+  process.stderr.write(
+    `Critical unit coverage dropped by ${Math.abs(comparison.coveredBranchDelta)} covered branches on the shared branch set.\n`
+  )
+  process.exit(1)
+}
+
+function parseOptions(args: string[]): Options {
+  const options: Partial<Options> = {}
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i]
+    const next = args[i + 1]
+
+    if (arg === '--base' && next) {
+      options.base = next
+      i++
+    } else if (arg.startsWith('--base=')) {
+      options.base = arg.slice('--base='.length)
+    } else if (arg === '--head' && next) {
+      options.head = next
+      i++
+    } else if (arg.startsWith('--head=')) {
+      options.head = arg.slice('--head='.length)
+    }
+  }
+
+  if (!options.base || !options.head) {
+    throw new Error(
+      'Usage: compareCriticalCoverage --base <json> --head <json>'
+    )
+  }
+
+  return {
+    base: options.base,
+    head: options.head
+  }
+}
+
+function formatComparison(comparison: CriticalCoverageComparison): string {
+  const passed = comparison.coveredBranchDelta >= 0
+  const lines = [
+    `## Critical Unit Coverage Gate: ${passed ? 'PASS' : 'FAIL'}`,
+    '',
+    `Base tested commit: \`${comparison.baseSha}\``,
+    `PR tested commit: \`${comparison.headSha}\``,
+    '',
+    '| Metric | Count |',
+    '|---|--:|',
+    `| Comparable critical branches | ${comparison.commonBranches} |`,
+    `| Covered in base | ${comparison.commonCoveredBranchesInBase} |`,
+    `| Covered in head | ${comparison.commonCoveredBranchesInHead} |`,
+    `| Covered branch delta | ${formatSignedCount(comparison.coveredBranchDelta)} |`,
+    `| Base-only branches | ${comparison.baseOnlyBranches} |`,
+    `| Head-only branches | ${comparison.headOnlyBranches} |`,
+    `| Covered-to-uncovered branches | ${comparison.regressions.length} |`,
+    ''
+  ]
+
+  if (passed) {
+    lines.push('PASS: Critical branch coverage did not decrease.')
+    return lines.join('\n')
+  }
+
+  lines.push('| File | Line | Branch | Base | Head |')
+  lines.push('|---|--:|---|--:|--:|')
+
+  for (const regression of comparison.regressions.slice(0, 25)) {
+    lines.push(
+      `| \`${regression.file}\` | ${regression.line} | ${regression.block}:${regression.branch} | ${formatTaken(regression.baseTaken)} | ${formatTaken(regression.headTaken)} |`
+    )
+  }
+
+  if (comparison.regressions.length > 25) {
+    lines.push('')
+    lines.push(
+      `${comparison.regressions.length - 25} additional regressions omitted from this summary.`
+    )
+  }
+
+  return lines.join('\n')
+}
+
+function formatTaken(value: number | null): string {
+  return value === null ? '0' : String(value)
+}
+
+function formatSignedCount(value: number): string {
+  return value > 0 ? `+${value}` : String(value)
+}

scripts/critical-coverage/criticalCoverageDirs.ts

@@ -0,0 +1,45 @@
+export const CRITICAL_COVERAGE_DIRS = [
+  'src/base',
+  'src/composables',
+  'src/core',
+  'src/lib/litegraph/src/node',
+  'src/lib/litegraph/src/subgraph',
+  'src/lib/litegraph/src/utils',
+  'src/platform/assets/composables',
+  'src/platform/assets/mappings',
+  'src/platform/assets/schemas',
+  'src/platform/assets/services',
+  'src/platform/assets/utils',
+  'src/platform/errorCatalog',
+  'src/platform/keybindings',
+  'src/platform/missingMedia',
+  'src/platform/missingModel',
+  'src/platform/navigation',
+  'src/platform/nodeReplacement',
+  'src/platform/remote',
+  'src/platform/remoteConfig',
+  'src/platform/secrets',
+  'src/platform/settings',
+  'src/platform/workflow',
+  'src/platform/workspace/api',
+  'src/platform/workspace/auth',
+  'src/platform/workspace/composables',
+  'src/platform/workspace/stores',
+  'src/platform/workspace/utils',
+  'src/schemas',
+  'src/scripts',
+  'src/services',
+  'src/stores',
+  'src/utils',
+  'src/workbench/extensions/manager/composables',
+  'src/workbench/extensions/manager/services',
+  'src/workbench/extensions/manager/stores',
+  'src/workbench/extensions/manager/utils',
+  'src/workbench/utils'
+] as const
+
+export function isCriticalCoveragePath(filePath: string): boolean {
+  return CRITICAL_COVERAGE_DIRS.some(
+    (dir) => filePath === dir || filePath.startsWith(`${dir}/`)
+  )
+}

scripts/critical-coverage/criticalCoverageReport.ts

@@ -0,0 +1,322 @@
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { dirname, isAbsolute, relative } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import {
+  CRITICAL_COVERAGE_DIRS,
+  isCriticalCoveragePath
+} from './criticalCoverageDirs'
+
+export interface CriticalBranchCoverage {
+  key: string
+  file: string
+  line: number
+  block: string
+  branch: string
+  taken: number | null
+  covered: boolean
+}
+
+export interface CriticalCoverageReport {
+  schemaVersion: 1
+  source: 'lcov'
+  sha: string
+  generatedAt: string
+  inputPath: string
+  criticalDirs: readonly string[]
+  totals: {
+    files: number
+    branches: number
+    coveredBranches: number
+  }
+  branches: CriticalBranchCoverage[]
+}
+
+export interface CriticalCoverageRegression extends CriticalBranchCoverage {
+  baseTaken: number | null
+  headTaken: number | null
+}
+
+export interface CriticalCoverageComparison {
+  baseSha: string
+  headSha: string
+  commonBranches: number
+  baseOnlyBranches: number
+  headOnlyBranches: number
+  commonCoveredBranchesInBase: number
+  commonCoveredBranchesInHead: number
+  coveredBranchDelta: number
+  regressions: CriticalCoverageRegression[]
+}
+
+interface CreateReportOptions {
+  inputPath: string
+  sha: string
+  generatedAt?: string
+  cwd?: string
+}
+
+export function createCriticalCoverageReport({
+  inputPath,
+  sha,
+  generatedAt = new Date().toISOString(),
+  cwd = process.cwd()
+}: CreateReportOptions): CriticalCoverageReport {
+  const lcov = readFileSync(inputPath, 'utf-8')
+  const branches = parseCriticalBranches(lcov, cwd)
+  const files = new Set(branches.map((branch) => branch.file))
+  const coveredBranches = branches.filter((branch) => branch.covered).length
+
+  return {
+    schemaVersion: 1,
+    source: 'lcov',
+    sha,
+    generatedAt,
+    inputPath,
+    criticalDirs: CRITICAL_COVERAGE_DIRS,
+    totals: {
+      files: files.size,
+      branches: branches.length,
+      coveredBranches
+    },
+    branches
+  }
+}
+
+export function writeCriticalCoverageReport(
+  report: CriticalCoverageReport,
+  outputPath: string
+): void {
+  mkdirSync(dirname(outputPath), { recursive: true })
+  writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`)
+}
+
+export function readCriticalCoverageReport(
+  inputPath: string
+): CriticalCoverageReport {
+  const parsed: unknown = JSON.parse(readFileSync(inputPath, 'utf-8'))
+
+  if (!isCriticalCoverageReport(parsed)) {
+    throw new Error(`Invalid critical coverage report: ${inputPath}`)
+  }
+
+  return parsed
+}
+
+export function compareCriticalCoverageReports(
+  base: CriticalCoverageReport,
+  head: CriticalCoverageReport
+): CriticalCoverageComparison {
+  const baseBranches = new Map(
+    base.branches.map((branch) => [branch.key, branch])
+  )
+  const headBranches = new Map(
+    head.branches.map((branch) => [branch.key, branch])
+  )
+  const regressions: CriticalCoverageRegression[] = []
+  let commonBranches = 0
+  let commonCoveredBranchesInBase = 0
+  let commonCoveredBranchesInHead = 0
+  let baseOnlyBranches = 0
+
+  for (const [key, baseBranch] of baseBranches) {
+    const headBranch = headBranches.get(key)
+
+    if (!headBranch) {
+      baseOnlyBranches++
+      continue
+    }
+
+    commonBranches++
+
+    if (baseBranch.covered) {
+      commonCoveredBranchesInBase++
+    }
+
+    if (headBranch.covered) {
+      commonCoveredBranchesInHead++
+    }
+
+    if (baseBranch.covered && !headBranch.covered) {
+      regressions.push({
+        ...baseBranch,
+        baseTaken: baseBranch.taken,
+        headTaken: headBranch.taken
+      })
+    }
+  }
+
+  return {
+    baseSha: base.sha,
+    headSha: head.sha,
+    commonBranches,
+    baseOnlyBranches,
+    headOnlyBranches: [...headBranches.keys()].filter(
+      (key) => !baseBranches.has(key)
+    ).length,
+    commonCoveredBranchesInBase,
+    commonCoveredBranchesInHead,
+    coveredBranchDelta:
+      commonCoveredBranchesInHead - commonCoveredBranchesInBase,
+    regressions: regressions.sort(compareBranches)
+  }
+}
+
+function parseCriticalBranches(
+  lcov: string,
+  cwd: string
+): CriticalBranchCoverage[] {
+  let currentFile = ''
+  const branches = new Map<string, CriticalBranchCoverage>()
+
+  for (const line of lcov.split('\n')) {
+    if (line.startsWith('SF:')) {
+      currentFile = normalizeCoveragePath(line.slice(3), cwd)
+      continue
+    }
+
+    if (!line.startsWith('BRDA:') || !isCriticalCoveragePath(currentFile)) {
+      continue
+    }
+
+    const branch = parseBranchData(currentFile, line.slice(5))
+    if (!branch) {
+      continue
+    }
+
+    const existing = branches.get(branch.key)
+    if (!existing) {
+      branches.set(branch.key, branch)
+      continue
+    }
+
+    branches.set(branch.key, mergeBranchCoverage(existing, branch))
+  }
+
+  return [...branches.values()].sort(compareBranches)
+}
+
+function parseBranchData(
+  file: string,
+  data: string
+): CriticalBranchCoverage | null {
+  const [lineValue, block, branch, takenValue] = data.split(',')
+  const line = Number(lineValue)
+
+  if (
+    !Number.isInteger(line) ||
+    !block ||
+    !branch ||
+    takenValue === undefined
+  ) {
+    return null
+  }
+
+  const taken = takenValue === '-' ? null : Number(takenValue)
+  const covered = taken !== null && Number.isFinite(taken) && taken > 0
+  const key = `${file}:${line}:${block}:${branch}`
+
+  return {
+    key,
+    file,
+    line,
+    block,
+    branch,
+    taken: Number.isFinite(taken) ? taken : null,
+    covered
+  }
+}
+
+function mergeBranchCoverage(
+  left: CriticalBranchCoverage,
+  right: CriticalBranchCoverage
+): CriticalBranchCoverage {
+  const taken =
+    left.taken === null || right.taken === null
+      ? null
+      : left.taken + right.taken
+
+  return {
+    ...left,
+    taken,
+    covered: left.covered || right.covered
+  }
+}
+
+function compareBranches(
+  left: CriticalBranchCoverage,
+  right: CriticalBranchCoverage
+): number {
+  return (
+    left.file.localeCompare(right.file) ||
+    left.line - right.line ||
+    left.block.localeCompare(right.block) ||
+    left.branch.localeCompare(right.branch)
+  )
+}
+
+function normalizeCoveragePath(filePath: string, cwd: string): string {
+  const decodedPath = filePath.startsWith('file://')
+    ? fileURLToPath(filePath)
+    : filePath
+
+  const relativePath = isAbsolute(decodedPath)
+    ? relative(cwd, decodedPath)
+    : decodedPath
+
+  const normalizedPath = relativePath.replace(/\\/g, '/').replace(/^\.\//, '')
+
+  if (!normalizedPath.startsWith('../')) {
+    return normalizedPath
+  }
+
+  const srcIndex = normalizedPath.indexOf('/src/')
+  return srcIndex === -1 ? normalizedPath : normalizedPath.slice(srcIndex + 1)
+}
+
+function isCriticalCoverageReport(
+  value: unknown
+): value is CriticalCoverageReport {
+  if (!isRecord(value)) {
+    return false
+  }
+
+  const totals = value.totals
+
+  return (
+    value.schemaVersion === 1 &&
+    value.source === 'lcov' &&
+    typeof value.sha === 'string' &&
+    typeof value.generatedAt === 'string' &&
+    typeof value.inputPath === 'string' &&
+    Array.isArray(value.criticalDirs) &&
+    isRecord(totals) &&
+    typeof totals.files === 'number' &&
+    typeof totals.branches === 'number' &&
+    typeof totals.coveredBranches === 'number' &&
+    Array.isArray(value.branches) &&
+    value.branches.every(isCriticalBranchCoverage)
+  )
+}
+
+function isCriticalBranchCoverage(
+  value: unknown
+): value is CriticalBranchCoverage {
+  if (!isRecord(value)) {
+    return false
+  }
+
+  return (
+    typeof value.key === 'string' &&
+    typeof value.file === 'string' &&
+    typeof value.line === 'number' &&
+    typeof value.block === 'string' &&
+    typeof value.branch === 'string' &&
+    (typeof value.taken === 'number' || value.taken === null) &&
+    typeof value.covered === 'boolean'
+  )
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null
+}

scripts/critical-coverage/extractCriticalCoverage.ts

@@ -0,0 +1,58 @@
+import {
+  createCriticalCoverageReport,
+  writeCriticalCoverageReport
+} from './criticalCoverageReport'
+
+interface Options {
+  input: string
+  output: string
+  sha: string
+}
+
+const options = parseOptions(process.argv.slice(2))
+const report = createCriticalCoverageReport({
+  inputPath: options.input,
+  sha: options.sha
+})
+
+writeCriticalCoverageReport(report, options.output)
+
+process.stdout.write(
+  [
+    `Critical coverage branches: ${report.totals.coveredBranches}/${report.totals.branches}`,
+    `Critical coverage files: ${report.totals.files}`,
+    `Wrote ${options.output}`
+  ].join('\n') + '\n'
+)
+
+function parseOptions(args: string[]): Options {
+  const options: Options = {
+    input: 'coverage/lcov.info',
+    output: 'coverage/critical-unit-coverage.json',
+    sha: process.env.GITHUB_SHA ?? 'unknown'
+  }
+
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i]
+    const next = args[i + 1]
+
+    if (arg === '--input' && next) {
+      options.input = next
+      i++
+    } else if (arg.startsWith('--input=')) {
+      options.input = arg.slice('--input='.length)
+    } else if (arg === '--output' && next) {
+      options.output = next
+      i++
+    } else if (arg.startsWith('--output=')) {
+      options.output = arg.slice('--output='.length)
+    } else if (arg === '--sha' && next) {
+      options.sha = next
+      i++
+    } else if (arg.startsWith('--sha=')) {
+      options.sha = arg.slice('--sha='.length)
+    }
+  }
+
+  return options
+}

vite.config.mts

@@ -18,6 +18,7 @@ import { createHtmlPlugin } from 'vite-plugin-html'
 import vueDevTools from 'vite-plugin-vue-devtools'
 
 import { comfyAPIPlugin } from './build/plugins'
+import { CRITICAL_COVERAGE_DIRS } from './scripts/critical-coverage/criticalCoverageDirs'
 
 dotenvConfig()
 
@@ -30,46 +31,6 @@ const DISABLE_TEMPLATES_PROXY = process.env.DISABLE_TEMPLATES_PROXY === 'true'
 const GENERATE_SOURCEMAP = process.env.GENERATE_SOURCEMAP !== 'false'
 const IS_STORYBOOK = process.env.npm_lifecycle_event === 'storybook'
 
-const CRITICAL_COVERAGE_DIRS = [
-  'src/base',
-  'src/composables',
-  'src/core',
-  'src/lib/litegraph/src/node',
-  'src/lib/litegraph/src/subgraph',
-  'src/lib/litegraph/src/utils',
-  'src/platform/assets/composables',
-  'src/platform/assets/mappings',
-  'src/platform/assets/schemas',
-  'src/platform/assets/services',
-  'src/platform/assets/utils',
-  'src/platform/errorCatalog',
-  'src/platform/keybindings',
-  'src/platform/missingMedia',
-  'src/platform/missingModel',
-  'src/platform/navigation',
-  'src/platform/nodeReplacement',
-  'src/platform/remote',
-  'src/platform/remoteConfig',
-  'src/platform/secrets',
-  'src/platform/settings',
-  'src/platform/workflow',
-  'src/platform/workspace/api',
-  'src/platform/workspace/auth',
-  'src/platform/workspace/composables',
-  'src/platform/workspace/stores',
-  'src/platform/workspace/utils',
-  'src/schemas',
-  'src/scripts',
-  'src/services',
-  'src/stores',
-  'src/utils',
-  'src/workbench/extensions/manager/composables',
-  'src/workbench/extensions/manager/services',
-  'src/workbench/extensions/manager/stores',
-  'src/workbench/extensions/manager/utils',
-  'src/workbench/utils'
-]
-
 // A single glob key so vitest aggregates all critical dirs into one
 // thresholds bucket instead of one bucket per glob
 const CRITICAL_COVERAGE_GLOB = `{${CRITICAL_COVERAGE_DIRS.join(',')}}/**/*.{ts,vue}`