mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-04-28 02:02:08 +00:00
feat: add performance testing infrastructure with CDP metrics (#9170)
## Summary Add a permanent, non-failing performance regression detection system using Chrome DevTools Protocol metrics, with automatic PR commenting. ## Changes - **What**: Performance testing infrastructure — `PerformanceHelper` fixture class using CDP `Performance.getMetrics` to collect `RecalcStyleCount`, `LayoutCount`, `LayoutDuration`, `TaskDuration`, `JSHeapUsedSize`. Adds `@perf` Playwright project (Chromium-only, single-threaded, 60s timeout), 4 baseline perf tests, CI workflow with sticky PR comment reporting, and `perf-report.js` script for generating markdown comparison tables. ## Review Focus - `PerformanceHelper` uses `page.context().newCDPSession(page)` — CDP is Chromium-only, so perf metrics are not collected on Firefox. This is intentional since CDP gives us browser-level style recalc/layout counts that `performance.mark/measure` cannot capture. - The CI workflow uses `continue-on-error: true` so perf tests never block merging. - Baseline comparison uses `dawidd6/action-download-artifact` to download metrics from the target branch, following the same pattern as `pr-size-report.yaml`. ## Stack This is the foundation PR for the Firefox performance fix stack: 1. **→ This PR: perf testing infrastructure** 2. `perf/fix-cursor-cache` — cursor style caching (depends on this) 3. `perf/fix-subgraph-svg` — SVG pre-rasterization (depends on this) 4. `perf/fix-clippath-raf` — RAF batching for clip-path (depends on this) PRs 2-4 are independent of each other. ┆Issue is synchronized with this [Notion page](https://www.notion.so/PR-9170-feat-add-performance-testing-infrastructure-with-CDP-metrics-3116d73d3650817cb43def6f8e9917f8) by [Unito](https://www.unito.io) --------- Co-authored-by: GitHub Action <action@github.com> Co-authored-by: Alexander Brown <drjkl@comfy.org>
This commit is contained in:
@@ -24,6 +24,7 @@ import {
|
||||
} from './components/SidebarTab'
|
||||
import { Topbar } from './components/Topbar'
|
||||
import { CanvasHelper } from './helpers/CanvasHelper'
|
||||
import { PerformanceHelper } from './helpers/PerformanceHelper'
|
||||
import { ClipboardHelper } from './helpers/ClipboardHelper'
|
||||
import { CommandHelper } from './helpers/CommandHelper'
|
||||
import { DragDropHelper } from './helpers/DragDropHelper'
|
||||
@@ -185,6 +186,7 @@ export class ComfyPage {
|
||||
public readonly dragDrop: DragDropHelper
|
||||
public readonly command: CommandHelper
|
||||
public readonly bottomPanel: BottomPanel
|
||||
public readonly perf: PerformanceHelper
|
||||
|
||||
/** Worker index to test user ID */
|
||||
public readonly userIds: string[] = []
|
||||
@@ -229,6 +231,7 @@ export class ComfyPage {
|
||||
this.dragDrop = new DragDropHelper(page, this.assetPath.bind(this))
|
||||
this.command = new CommandHelper(page)
|
||||
this.bottomPanel = new BottomPanel(page)
|
||||
this.perf = new PerformanceHelper(page)
|
||||
}
|
||||
|
||||
get visibleToasts() {
|
||||
@@ -436,7 +439,13 @@ export const comfyPageFixture = base.extend<{
|
||||
}
|
||||
|
||||
await comfyPage.setup()
|
||||
|
||||
const isPerf = testInfo.tags.includes('@perf')
|
||||
if (isPerf) await comfyPage.perf.init()
|
||||
|
||||
await use(comfyPage)
|
||||
|
||||
if (isPerf) await comfyPage.perf.dispose()
|
||||
},
|
||||
comfyMouse: async ({ comfyPage }, use) => {
|
||||
const comfyMouse = new ComfyMouse(comfyPage)
|
||||
|
||||
96
browser_tests/fixtures/helpers/PerformanceHelper.ts
Normal file
96
browser_tests/fixtures/helpers/PerformanceHelper.ts
Normal file
@@ -0,0 +1,96 @@
|
||||
import type { CDPSession, Page } from '@playwright/test'
|
||||
|
||||
interface PerfSnapshot {
|
||||
RecalcStyleCount: number
|
||||
RecalcStyleDuration: number
|
||||
LayoutCount: number
|
||||
LayoutDuration: number
|
||||
TaskDuration: number
|
||||
JSHeapUsedSize: number
|
||||
Timestamp: number
|
||||
}
|
||||
|
||||
export interface PerfMeasurement {
|
||||
name: string
|
||||
durationMs: number
|
||||
styleRecalcs: number
|
||||
styleRecalcDurationMs: number
|
||||
layouts: number
|
||||
layoutDurationMs: number
|
||||
taskDurationMs: number
|
||||
heapDeltaBytes: number
|
||||
}
|
||||
|
||||
export class PerformanceHelper {
|
||||
private cdp: CDPSession | null = null
|
||||
private snapshot: PerfSnapshot | null = null
|
||||
|
||||
constructor(private readonly page: Page) {}
|
||||
|
||||
async init(): Promise<void> {
|
||||
this.cdp = await this.page.context().newCDPSession(this.page)
|
||||
await this.cdp.send('Performance.enable')
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
this.snapshot = null
|
||||
if (this.cdp) {
|
||||
try {
|
||||
await this.cdp.send('Performance.disable')
|
||||
} finally {
|
||||
await this.cdp.detach()
|
||||
this.cdp = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async getSnapshot(): Promise<PerfSnapshot> {
|
||||
if (!this.cdp) throw new Error('PerformanceHelper not initialized')
|
||||
const { metrics } = (await this.cdp.send('Performance.getMetrics')) as {
|
||||
metrics: { name: string; value: number }[]
|
||||
}
|
||||
function get(name: string): number {
|
||||
return metrics.find((m) => m.name === name)?.value ?? 0
|
||||
}
|
||||
return {
|
||||
RecalcStyleCount: get('RecalcStyleCount'),
|
||||
RecalcStyleDuration: get('RecalcStyleDuration'),
|
||||
LayoutCount: get('LayoutCount'),
|
||||
LayoutDuration: get('LayoutDuration'),
|
||||
TaskDuration: get('TaskDuration'),
|
||||
JSHeapUsedSize: get('JSHeapUsedSize'),
|
||||
Timestamp: get('Timestamp')
|
||||
}
|
||||
}
|
||||
|
||||
async startMeasuring(): Promise<void> {
|
||||
if (this.snapshot) {
|
||||
throw new Error(
|
||||
'Measurement already in progress — call stopMeasuring() first'
|
||||
)
|
||||
}
|
||||
this.snapshot = await this.getSnapshot()
|
||||
}
|
||||
|
||||
async stopMeasuring(name: string): Promise<PerfMeasurement> {
|
||||
if (!this.snapshot) throw new Error('Call startMeasuring() first')
|
||||
const after = await this.getSnapshot()
|
||||
const before = this.snapshot
|
||||
this.snapshot = null
|
||||
|
||||
function delta(key: keyof PerfSnapshot): number {
|
||||
return after[key] - before[key]
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
durationMs: delta('Timestamp') * 1000,
|
||||
styleRecalcs: delta('RecalcStyleCount'),
|
||||
styleRecalcDurationMs: delta('RecalcStyleDuration') * 1000,
|
||||
layouts: delta('LayoutCount'),
|
||||
layoutDurationMs: delta('LayoutDuration') * 1000,
|
||||
taskDurationMs: delta('TaskDuration') * 1000,
|
||||
heapDeltaBytes: delta('JSHeapUsedSize')
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,14 @@
|
||||
import type { FullConfig } from '@playwright/test'
|
||||
import dotenv from 'dotenv'
|
||||
|
||||
import { writePerfReport } from './helpers/perfReporter'
|
||||
import { restorePath } from './utils/backupUtils'
|
||||
|
||||
dotenv.config()
|
||||
|
||||
export default function globalTeardown(_config: FullConfig) {
|
||||
writePerfReport()
|
||||
|
||||
if (!process.env.CI && process.env.TEST_COMFYUI_DIR) {
|
||||
restorePath([process.env.TEST_COMFYUI_DIR, 'user'])
|
||||
restorePath([process.env.TEST_COMFYUI_DIR, 'models'])
|
||||
|
||||
49
browser_tests/helpers/perfReporter.ts
Normal file
49
browser_tests/helpers/perfReporter.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'fs'
|
||||
import { join } from 'path'
|
||||
|
||||
import type { PerfMeasurement } from '../fixtures/helpers/PerformanceHelper'
|
||||
|
||||
export interface PerfReport {
|
||||
timestamp: string
|
||||
gitSha: string
|
||||
branch: string
|
||||
measurements: PerfMeasurement[]
|
||||
}
|
||||
|
||||
const TEMP_DIR = join('test-results', 'perf-temp')
|
||||
|
||||
export function recordMeasurement(m: PerfMeasurement) {
|
||||
mkdirSync(TEMP_DIR, { recursive: true })
|
||||
const filename = `${m.name}-${Date.now()}.json`
|
||||
writeFileSync(join(TEMP_DIR, filename), JSON.stringify(m))
|
||||
}
|
||||
|
||||
export function writePerfReport(
|
||||
gitSha = process.env.GITHUB_SHA ?? 'local',
|
||||
branch = process.env.GITHUB_HEAD_REF ?? 'local'
|
||||
) {
|
||||
if (!readdirSync('test-results', { withFileTypes: true }).length) return
|
||||
|
||||
let tempFiles: string[]
|
||||
try {
|
||||
tempFiles = readdirSync(TEMP_DIR).filter((f) => f.endsWith('.json'))
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
if (tempFiles.length === 0) return
|
||||
|
||||
const measurements: PerfMeasurement[] = tempFiles.map((f) =>
|
||||
JSON.parse(readFileSync(join(TEMP_DIR, f), 'utf-8'))
|
||||
)
|
||||
|
||||
const report: PerfReport = {
|
||||
timestamp: new Date().toISOString(),
|
||||
gitSha,
|
||||
branch,
|
||||
measurements
|
||||
}
|
||||
writeFileSync(
|
||||
join('test-results', 'perf-metrics.json'),
|
||||
JSON.stringify(report, null, 2)
|
||||
)
|
||||
}
|
||||
70
browser_tests/tests/performance.spec.ts
Normal file
70
browser_tests/tests/performance.spec.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
import { comfyPageFixture as test } from '../fixtures/ComfyPage'
|
||||
import { recordMeasurement } from '../helpers/perfReporter'
|
||||
|
||||
test.describe('Performance', { tag: ['@perf'] }, () => {
|
||||
test('canvas idle style recalculations', async ({ comfyPage }) => {
|
||||
await comfyPage.workflow.loadWorkflow('default')
|
||||
await comfyPage.perf.startMeasuring()
|
||||
|
||||
// Let the canvas idle for 2 seconds — no user interaction.
|
||||
// Measures baseline style recalcs from reactive state + render loop.
|
||||
for (let i = 0; i < 120; i++) {
|
||||
await comfyPage.nextFrame()
|
||||
}
|
||||
|
||||
const m = await comfyPage.perf.stopMeasuring('canvas-idle')
|
||||
recordMeasurement(m)
|
||||
console.log(
|
||||
`Canvas idle: ${m.styleRecalcs} style recalcs, ${m.layouts} layouts`
|
||||
)
|
||||
})
|
||||
|
||||
test('canvas mouse interaction style recalculations', async ({
|
||||
comfyPage
|
||||
}) => {
|
||||
await comfyPage.workflow.loadWorkflow('default')
|
||||
await comfyPage.perf.startMeasuring()
|
||||
|
||||
const canvas = comfyPage.canvas
|
||||
const box = await canvas.boundingBox()
|
||||
if (!box) throw new Error('Canvas bounding box not available')
|
||||
|
||||
// Sweep mouse across the canvas — crosses nodes, empty space, slots
|
||||
for (let i = 0; i < 100; i++) {
|
||||
await comfyPage.page.mouse.move(
|
||||
box.x + (box.width * i) / 100,
|
||||
box.y + (box.height * (i % 3)) / 3
|
||||
)
|
||||
}
|
||||
|
||||
const m = await comfyPage.perf.stopMeasuring('canvas-mouse-sweep')
|
||||
recordMeasurement(m)
|
||||
console.log(
|
||||
`Mouse sweep: ${m.styleRecalcs} style recalcs, ${m.layouts} layouts`
|
||||
)
|
||||
})
|
||||
|
||||
test('DOM widget clipping during node selection', async ({ comfyPage }) => {
|
||||
// Load default workflow which has DOM widgets (text inputs, combos)
|
||||
await comfyPage.workflow.loadWorkflow('default')
|
||||
await comfyPage.perf.startMeasuring()
|
||||
|
||||
// Select and deselect nodes rapidly to trigger clipping recalculation
|
||||
const canvas = comfyPage.canvas
|
||||
const box = await canvas.boundingBox()
|
||||
if (!box) throw new Error('Canvas bounding box not available')
|
||||
|
||||
for (let i = 0; i < 20; i++) {
|
||||
// Click on canvas area (nodes occupy various positions)
|
||||
await comfyPage.page.mouse.click(
|
||||
box.x + box.width / 3 + (i % 5) * 30,
|
||||
box.y + box.height / 3 + (i % 4) * 30
|
||||
)
|
||||
await comfyPage.nextFrame()
|
||||
}
|
||||
|
||||
const m = await comfyPage.perf.stopMeasuring('dom-widget-clipping')
|
||||
recordMeasurement(m)
|
||||
console.log(`Clipping: ${m.layouts} forced layouts`)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user