From 8e215b3174b40c1469215b841165d78d2ca3caf0 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 25 Feb 2026 20:09:57 -0800 Subject: [PATCH] feat: add performance testing infrastructure with CDP metrics (#9170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Add a permanent, non-failing performance regression detection system using Chrome DevTools Protocol metrics, with automatic PR commenting. ## Changes - **What**: Performance testing infrastructure — `PerformanceHelper` fixture class using CDP `Performance.getMetrics` to collect `RecalcStyleCount`, `LayoutCount`, `LayoutDuration`, `TaskDuration`, `JSHeapUsedSize`. Adds `@perf` Playwright project (Chromium-only, single-threaded, 60s timeout), 4 baseline perf tests, CI workflow with sticky PR comment reporting, and `perf-report.js` script for generating markdown comparison tables. ## Review Focus - `PerformanceHelper` uses `page.context().newCDPSession(page)` — CDP is Chromium-only, so perf metrics are not collected on Firefox. This is intentional since CDP gives us browser-level style recalc/layout counts that `performance.mark/measure` cannot capture. - The CI workflow uses `continue-on-error: true` so perf tests never block merging. - Baseline comparison uses `dawidd6/action-download-artifact` to download metrics from the target branch, following the same pattern as `pr-size-report.yaml`. ## Stack This is the foundation PR for the Firefox performance fix stack: 1. **→ This PR: perf testing infrastructure** 2. `perf/fix-cursor-cache` — cursor style caching (depends on this) 3. `perf/fix-subgraph-svg` — SVG pre-rasterization (depends on this) 4. `perf/fix-clippath-raf` — RAF batching for clip-path (depends on this) PRs 2-4 are independent of each other. ┆Issue is synchronized with this [Notion page](https://www.notion.so/PR-9170-feat-add-performance-testing-infrastructure-with-CDP-metrics-3116d73d3650817cb43def6f8e9917f8) by [Unito](https://www.unito.io) --------- Co-authored-by: GitHub Action Co-authored-by: Alexander Brown --- .github/workflows/ci-perf-report.yaml | 110 +++++++++++++++ browser_tests/fixtures/ComfyPage.ts | 9 ++ .../fixtures/helpers/PerformanceHelper.ts | 96 ++++++++++++++ browser_tests/globalTeardown.ts | 3 + browser_tests/helpers/perfReporter.ts | 49 +++++++ browser_tests/tests/performance.spec.ts | 70 ++++++++++ playwright.config.ts | 13 +- scripts/perf-report.ts | 125 ++++++++++++++++++ 8 files changed, 474 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci-perf-report.yaml create mode 100644 browser_tests/fixtures/helpers/PerformanceHelper.ts create mode 100644 browser_tests/helpers/perfReporter.ts create mode 100644 browser_tests/tests/performance.spec.ts create mode 100644 scripts/perf-report.ts diff --git a/.github/workflows/ci-perf-report.yaml b/.github/workflows/ci-perf-report.yaml new file mode 100644 index 0000000000..88640f1c70 --- /dev/null +++ b/.github/workflows/ci-perf-report.yaml @@ -0,0 +1,110 @@ +name: 'CI: Performance Report' + +on: + push: + branches: [main, core/*] + paths-ignore: ['**/*.md'] + pull_request: + branches-ignore: [wip/*, draft/*, temp/*] + paths-ignore: ['**/*.md'] + +concurrency: + group: perf-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + +jobs: + perf-tests: + if: github.repository == 'Comfy-Org/ComfyUI_frontend' + runs-on: ubuntu-latest + timeout-minutes: 30 + container: + image: ghcr.io/comfy-org/comfyui-ci-container:0.0.12 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: read + packages: read + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup frontend + uses: ./.github/actions/setup-frontend + with: + include_build_step: true + + - name: Start ComfyUI server + uses: ./.github/actions/start-comfyui-server + + - name: Run performance tests + id: perf + continue-on-error: true + run: pnpm exec playwright test --project=performance --workers=1 + + - name: Upload perf metrics + if: always() + uses: actions/upload-artifact@v6 + with: + name: perf-metrics + path: test-results/perf-metrics.json + retention-days: 30 + if-no-files-found: warn + + report: + needs: perf-tests + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: 22 + + - name: Download PR perf metrics + continue-on-error: true + uses: actions/download-artifact@v7 + with: + name: perf-metrics + path: test-results/ + + - name: Download baseline perf metrics + uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12 + with: + branch: ${{ github.event.pull_request.base.ref }} + workflow: ci-perf-report.yaml + event: push + name: perf-metrics + path: temp/perf-baseline/ + if_no_artifact_found: warn + + - name: Generate perf report + run: npx --yes tsx scripts/perf-report.ts > perf-report.md + + - name: Read perf report + id: perf-report + uses: juliangruber/read-file-action@b549046febe0fe86f8cb4f93c24e284433f9ab58 # v1.1.7 + with: + path: ./perf-report.md + + - name: Create or update PR comment + uses: actions-cool/maintain-one-comment@4b2dbf086015f892dcb5e8c1106f5fccd6c1476b # v3.2.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + number: ${{ github.event.pull_request.number }} + body: | + ${{ steps.perf-report.outputs.content }} + + body-include: '' diff --git a/browser_tests/fixtures/ComfyPage.ts b/browser_tests/fixtures/ComfyPage.ts index c4126c38c7..f0eaf29c6f 100644 --- a/browser_tests/fixtures/ComfyPage.ts +++ b/browser_tests/fixtures/ComfyPage.ts @@ -24,6 +24,7 @@ import { } from './components/SidebarTab' import { Topbar } from './components/Topbar' import { CanvasHelper } from './helpers/CanvasHelper' +import { PerformanceHelper } from './helpers/PerformanceHelper' import { ClipboardHelper } from './helpers/ClipboardHelper' import { CommandHelper } from './helpers/CommandHelper' import { DragDropHelper } from './helpers/DragDropHelper' @@ -185,6 +186,7 @@ export class ComfyPage { public readonly dragDrop: DragDropHelper public readonly command: CommandHelper public readonly bottomPanel: BottomPanel + public readonly perf: PerformanceHelper /** Worker index to test user ID */ public readonly userIds: string[] = [] @@ -229,6 +231,7 @@ export class ComfyPage { this.dragDrop = new DragDropHelper(page, this.assetPath.bind(this)) this.command = new CommandHelper(page) this.bottomPanel = new BottomPanel(page) + this.perf = new PerformanceHelper(page) } get visibleToasts() { @@ -436,7 +439,13 @@ export const comfyPageFixture = base.extend<{ } await comfyPage.setup() + + const isPerf = testInfo.tags.includes('@perf') + if (isPerf) await comfyPage.perf.init() + await use(comfyPage) + + if (isPerf) await comfyPage.perf.dispose() }, comfyMouse: async ({ comfyPage }, use) => { const comfyMouse = new ComfyMouse(comfyPage) diff --git a/browser_tests/fixtures/helpers/PerformanceHelper.ts b/browser_tests/fixtures/helpers/PerformanceHelper.ts new file mode 100644 index 0000000000..f02a03f21a --- /dev/null +++ b/browser_tests/fixtures/helpers/PerformanceHelper.ts @@ -0,0 +1,96 @@ +import type { CDPSession, Page } from '@playwright/test' + +interface PerfSnapshot { + RecalcStyleCount: number + RecalcStyleDuration: number + LayoutCount: number + LayoutDuration: number + TaskDuration: number + JSHeapUsedSize: number + Timestamp: number +} + +export interface PerfMeasurement { + name: string + durationMs: number + styleRecalcs: number + styleRecalcDurationMs: number + layouts: number + layoutDurationMs: number + taskDurationMs: number + heapDeltaBytes: number +} + +export class PerformanceHelper { + private cdp: CDPSession | null = null + private snapshot: PerfSnapshot | null = null + + constructor(private readonly page: Page) {} + + async init(): Promise { + this.cdp = await this.page.context().newCDPSession(this.page) + await this.cdp.send('Performance.enable') + } + + async dispose(): Promise { + this.snapshot = null + if (this.cdp) { + try { + await this.cdp.send('Performance.disable') + } finally { + await this.cdp.detach() + this.cdp = null + } + } + } + + private async getSnapshot(): Promise { + if (!this.cdp) throw new Error('PerformanceHelper not initialized') + const { metrics } = (await this.cdp.send('Performance.getMetrics')) as { + metrics: { name: string; value: number }[] + } + function get(name: string): number { + return metrics.find((m) => m.name === name)?.value ?? 0 + } + return { + RecalcStyleCount: get('RecalcStyleCount'), + RecalcStyleDuration: get('RecalcStyleDuration'), + LayoutCount: get('LayoutCount'), + LayoutDuration: get('LayoutDuration'), + TaskDuration: get('TaskDuration'), + JSHeapUsedSize: get('JSHeapUsedSize'), + Timestamp: get('Timestamp') + } + } + + async startMeasuring(): Promise { + if (this.snapshot) { + throw new Error( + 'Measurement already in progress — call stopMeasuring() first' + ) + } + this.snapshot = await this.getSnapshot() + } + + async stopMeasuring(name: string): Promise { + if (!this.snapshot) throw new Error('Call startMeasuring() first') + const after = await this.getSnapshot() + const before = this.snapshot + this.snapshot = null + + function delta(key: keyof PerfSnapshot): number { + return after[key] - before[key] + } + + return { + name, + durationMs: delta('Timestamp') * 1000, + styleRecalcs: delta('RecalcStyleCount'), + styleRecalcDurationMs: delta('RecalcStyleDuration') * 1000, + layouts: delta('LayoutCount'), + layoutDurationMs: delta('LayoutDuration') * 1000, + taskDurationMs: delta('TaskDuration') * 1000, + heapDeltaBytes: delta('JSHeapUsedSize') + } + } +} diff --git a/browser_tests/globalTeardown.ts b/browser_tests/globalTeardown.ts index c69f563df6..38dc26a3d2 100644 --- a/browser_tests/globalTeardown.ts +++ b/browser_tests/globalTeardown.ts @@ -1,11 +1,14 @@ import type { FullConfig } from '@playwright/test' import dotenv from 'dotenv' +import { writePerfReport } from './helpers/perfReporter' import { restorePath } from './utils/backupUtils' dotenv.config() export default function globalTeardown(_config: FullConfig) { + writePerfReport() + if (!process.env.CI && process.env.TEST_COMFYUI_DIR) { restorePath([process.env.TEST_COMFYUI_DIR, 'user']) restorePath([process.env.TEST_COMFYUI_DIR, 'models']) diff --git a/browser_tests/helpers/perfReporter.ts b/browser_tests/helpers/perfReporter.ts new file mode 100644 index 0000000000..2fac7cac5e --- /dev/null +++ b/browser_tests/helpers/perfReporter.ts @@ -0,0 +1,49 @@ +import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'fs' +import { join } from 'path' + +import type { PerfMeasurement } from '../fixtures/helpers/PerformanceHelper' + +export interface PerfReport { + timestamp: string + gitSha: string + branch: string + measurements: PerfMeasurement[] +} + +const TEMP_DIR = join('test-results', 'perf-temp') + +export function recordMeasurement(m: PerfMeasurement) { + mkdirSync(TEMP_DIR, { recursive: true }) + const filename = `${m.name}-${Date.now()}.json` + writeFileSync(join(TEMP_DIR, filename), JSON.stringify(m)) +} + +export function writePerfReport( + gitSha = process.env.GITHUB_SHA ?? 'local', + branch = process.env.GITHUB_HEAD_REF ?? 'local' +) { + if (!readdirSync('test-results', { withFileTypes: true }).length) return + + let tempFiles: string[] + try { + tempFiles = readdirSync(TEMP_DIR).filter((f) => f.endsWith('.json')) + } catch { + return + } + if (tempFiles.length === 0) return + + const measurements: PerfMeasurement[] = tempFiles.map((f) => + JSON.parse(readFileSync(join(TEMP_DIR, f), 'utf-8')) + ) + + const report: PerfReport = { + timestamp: new Date().toISOString(), + gitSha, + branch, + measurements + } + writeFileSync( + join('test-results', 'perf-metrics.json'), + JSON.stringify(report, null, 2) + ) +} diff --git a/browser_tests/tests/performance.spec.ts b/browser_tests/tests/performance.spec.ts new file mode 100644 index 0000000000..a77a559aad --- /dev/null +++ b/browser_tests/tests/performance.spec.ts @@ -0,0 +1,70 @@ +import { comfyPageFixture as test } from '../fixtures/ComfyPage' +import { recordMeasurement } from '../helpers/perfReporter' + +test.describe('Performance', { tag: ['@perf'] }, () => { + test('canvas idle style recalculations', async ({ comfyPage }) => { + await comfyPage.workflow.loadWorkflow('default') + await comfyPage.perf.startMeasuring() + + // Let the canvas idle for 2 seconds — no user interaction. + // Measures baseline style recalcs from reactive state + render loop. + for (let i = 0; i < 120; i++) { + await comfyPage.nextFrame() + } + + const m = await comfyPage.perf.stopMeasuring('canvas-idle') + recordMeasurement(m) + console.log( + `Canvas idle: ${m.styleRecalcs} style recalcs, ${m.layouts} layouts` + ) + }) + + test('canvas mouse interaction style recalculations', async ({ + comfyPage + }) => { + await comfyPage.workflow.loadWorkflow('default') + await comfyPage.perf.startMeasuring() + + const canvas = comfyPage.canvas + const box = await canvas.boundingBox() + if (!box) throw new Error('Canvas bounding box not available') + + // Sweep mouse across the canvas — crosses nodes, empty space, slots + for (let i = 0; i < 100; i++) { + await comfyPage.page.mouse.move( + box.x + (box.width * i) / 100, + box.y + (box.height * (i % 3)) / 3 + ) + } + + const m = await comfyPage.perf.stopMeasuring('canvas-mouse-sweep') + recordMeasurement(m) + console.log( + `Mouse sweep: ${m.styleRecalcs} style recalcs, ${m.layouts} layouts` + ) + }) + + test('DOM widget clipping during node selection', async ({ comfyPage }) => { + // Load default workflow which has DOM widgets (text inputs, combos) + await comfyPage.workflow.loadWorkflow('default') + await comfyPage.perf.startMeasuring() + + // Select and deselect nodes rapidly to trigger clipping recalculation + const canvas = comfyPage.canvas + const box = await canvas.boundingBox() + if (!box) throw new Error('Canvas bounding box not available') + + for (let i = 0; i < 20; i++) { + // Click on canvas area (nodes occupy various positions) + await comfyPage.page.mouse.click( + box.x + box.width / 3 + (i % 5) * 30, + box.y + box.height / 3 + (i % 4) * 30 + ) + await comfyPage.nextFrame() + } + + const m = await comfyPage.perf.stopMeasuring('dom-widget-clipping') + recordMeasurement(m) + console.log(`Clipping: ${m.layouts} forced layouts`) + }) +}) diff --git a/playwright.config.ts b/playwright.config.ts index e080f0c85a..cf5d199370 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -36,7 +36,18 @@ export default defineConfig({ name: 'chromium', use: { ...devices['Desktop Chrome'] }, timeout: 15000, - grepInvert: /@mobile/ // Run all tests except those tagged with @mobile + grepInvert: /@mobile|@perf/ // Run all tests except those tagged with @mobile or @perf + }, + + { + name: 'performance', + use: { + ...devices['Desktop Chrome'], + trace: 'retain-on-failure' + }, + timeout: 60_000, + grep: /@perf/, + fullyParallel: false }, { diff --git a/scripts/perf-report.ts b/scripts/perf-report.ts new file mode 100644 index 0000000000..b8d288f101 --- /dev/null +++ b/scripts/perf-report.ts @@ -0,0 +1,125 @@ +import { existsSync, readFileSync } from 'node:fs' + +interface PerfMeasurement { + name: string + durationMs: number + styleRecalcs: number + styleRecalcDurationMs: number + layouts: number + layoutDurationMs: number + taskDurationMs: number + heapDeltaBytes: number +} + +interface PerfReport { + timestamp: string + gitSha: string + branch: string + measurements: PerfMeasurement[] +} + +const CURRENT_PATH = 'test-results/perf-metrics.json' +const BASELINE_PATH = 'temp/perf-baseline/perf-metrics.json' + +function formatDelta(pct: number): string { + if (pct >= 20) return `+${pct.toFixed(0)}% 🔴` + if (pct >= 10) return `+${pct.toFixed(0)}% 🟠` + if (pct > -10) return `${pct >= 0 ? '+' : ''}${pct.toFixed(0)}% ⚪` + return `${pct.toFixed(0)}% 🟢` +} + +function formatBytes(bytes: number): string { + if (Math.abs(bytes) < 1024) return `${bytes} B` + if (Math.abs(bytes) < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB` + return `${(bytes / (1024 * 1024)).toFixed(1)} MB` +} + +function calcDelta( + baseline: number, + current: number +): { pct: number; isNew: boolean } { + if (baseline > 0) { + return { pct: ((current - baseline) / baseline) * 100, isNew: false } + } + return current > 0 ? { pct: Infinity, isNew: true } : { pct: 0, isNew: false } +} + +function formatDeltaCell(delta: { pct: number; isNew: boolean }): string { + return delta.isNew ? 'new 🔴' : formatDelta(delta.pct) +} + +function main() { + if (!existsSync(CURRENT_PATH)) { + process.stdout.write( + '## ⚡ Performance Report\n\nNo perf metrics found. Perf tests may not have run.\n' + ) + process.exit(0) + } + + const current: PerfReport = JSON.parse(readFileSync(CURRENT_PATH, 'utf-8')) + + const baseline: PerfReport | null = existsSync(BASELINE_PATH) + ? JSON.parse(readFileSync(BASELINE_PATH, 'utf-8')) + : null + + const lines: string[] = [] + lines.push('## ⚡ Performance Report\n') + + if (baseline) { + lines.push( + '| Metric | Baseline | PR | Δ |', + '|--------|----------|-----|---|' + ) + + for (const m of current.measurements) { + const base = baseline.measurements.find((b) => b.name === m.name) + if (!base) { + lines.push(`| ${m.name}: style recalcs | — | ${m.styleRecalcs} | new |`) + lines.push(`| ${m.name}: layouts | — | ${m.layouts} | new |`) + lines.push( + `| ${m.name}: task duration | — | ${m.taskDurationMs.toFixed(0)}ms | new |` + ) + continue + } + + const recalcDelta = calcDelta(base.styleRecalcs, m.styleRecalcs) + lines.push( + `| ${m.name}: style recalcs | ${base.styleRecalcs} | ${m.styleRecalcs} | ${formatDeltaCell(recalcDelta)} |` + ) + + const layoutDelta = calcDelta(base.layouts, m.layouts) + lines.push( + `| ${m.name}: layouts | ${base.layouts} | ${m.layouts} | ${formatDeltaCell(layoutDelta)} |` + ) + + const taskDelta = calcDelta(base.taskDurationMs, m.taskDurationMs) + lines.push( + `| ${m.name}: task duration | ${base.taskDurationMs.toFixed(0)}ms | ${m.taskDurationMs.toFixed(0)}ms | ${formatDeltaCell(taskDelta)} |` + ) + } + } else { + lines.push( + 'No baseline found — showing absolute values.\n', + '| Metric | Value |', + '|--------|-------|' + ) + for (const m of current.measurements) { + lines.push(`| ${m.name}: style recalcs | ${m.styleRecalcs} |`) + lines.push(`| ${m.name}: layouts | ${m.layouts} |`) + lines.push( + `| ${m.name}: task duration | ${m.taskDurationMs.toFixed(0)}ms |` + ) + lines.push(`| ${m.name}: heap delta | ${formatBytes(m.heapDeltaBytes)} |`) + } + } + + lines.push('\n
Raw data\n') + lines.push('```json') + lines.push(JSON.stringify(current, null, 2)) + lines.push('```') + lines.push('\n
') + + process.stdout.write(lines.join('\n') + '\n') +} + +main()