diff --git a/browser_tests/fixtures/ComfyPage.ts b/browser_tests/fixtures/ComfyPage.ts index 2f6fc0b14..c30bc55e7 100644 --- a/browser_tests/fixtures/ComfyPage.ts +++ b/browser_tests/fixtures/ComfyPage.ts @@ -10,6 +10,7 @@ import type { KeyCombo } from '../../src/schemas/keyBindingSchema' import type { useWorkspaceStore } from '../../src/stores/workspaceStore' import { NodeBadgeMode } from '../../src/types/nodeSource' import { ComfyActionbar } from '../helpers/actionbar' +import { PerformanceMonitor } from '../helpers/performanceMonitor' import { ComfyTemplates } from '../helpers/templates' import { ComfyMouse } from './ComfyMouse' import { ComfyNodeSearchBox } from './components/ComfyNodeSearchBox' @@ -143,6 +144,7 @@ export class ComfyPage { public readonly templates: ComfyTemplates public readonly settingDialog: SettingDialog public readonly confirmDialog: ConfirmDialog + public readonly performanceMonitor: PerformanceMonitor /** Worker index to test user ID */ public readonly userIds: string[] = [] @@ -170,6 +172,7 @@ export class ComfyPage { this.templates = new ComfyTemplates(page) this.settingDialog = new SettingDialog(page) this.confirmDialog = new ConfirmDialog(page) + this.performanceMonitor = new PerformanceMonitor(page) } convertLeafToContent(structure: FolderStructure): FolderStructure { @@ -762,7 +765,7 @@ export class ComfyPage { y: 625 } }) - this.page.mouse.move(10, 10) + await this.page.mouse.move(10, 10) await this.nextFrame() } @@ -774,7 +777,7 @@ export class ComfyPage { }, button: 'right' }) - this.page.mouse.move(10, 10) + await this.page.mouse.move(10, 10) await this.nextFrame() } @@ -1058,6 +1061,14 @@ export const comfyPageFixture = base.extend<{ const userId = await comfyPage.setupUser(username) comfyPage.userIds[parallelIndex] = userId + // Enable performance monitoring for @perf tagged tests + const isPerformanceTest = testInfo.title.includes('@perf') + // console.log('test info', testInfo) + if (isPerformanceTest) { + console.log('Enabling performance monitoring') + // PerformanceMonitor.enable() + } + try { await comfyPage.setupSettings({ 'Comfy.UseNewMenu': 'Disabled', @@ -1078,12 +1089,24 @@ export const comfyPageFixture = base.extend<{ console.error(e) } + if (isPerformanceTest) { + // Start performance monitoring just before test execution + console.log('Starting performance monitoring') + await comfyPage.performanceMonitor.startMonitoring(testInfo.title) + } + await comfyPage.setup() await use(comfyPage) + + // Cleanup performance monitoring and collect final metrics + if (isPerformanceTest) { + console.log('Finishing performance monitoring') + await comfyPage.performanceMonitor.finishMonitoring(testInfo.title) + } }, comfyMouse: async ({ comfyPage }, use) => { const comfyMouse = new ComfyMouse(comfyPage) - use(comfyMouse) + void use(comfyMouse) } }) diff --git a/browser_tests/helpers/performanceMonitor.ts b/browser_tests/helpers/performanceMonitor.ts new file mode 100644 index 000000000..1f58d9362 --- /dev/null +++ b/browser_tests/helpers/performanceMonitor.ts @@ -0,0 +1,206 @@ +import type { Page } from '@playwright/test' + +export interface PerformanceMetrics { + testName: string + timestamp: number + branch?: string + memoryUsage: { + usedJSHeapSize: number + totalJSHeapSize: number + jsHeapSizeLimit: number + } | null + timing: { + loadStart?: number + domContentLoaded?: number + loadComplete?: number + firstPaint?: number + firstContentfulPaint?: number + largestContentfulPaint?: number + } + customMetrics: Record +} + +export class PerformanceMonitor { + private metrics: PerformanceMetrics[] = [] + + constructor(private page: Page) {} + + async startMonitoring(testName: string) { + await this.page.evaluate((testName) => { + // Initialize performance monitoring + window.perfMonitor = { + testName, + startTime: performance.now(), + marks: new Map(), + measures: new Map() + } + + // Mark test start + performance.mark(`${testName}-start`) + + // Set up performance observer to capture paint metrics + if ('PerformanceObserver' in window) { + const observer = new PerformanceObserver((list) => { + for (const entry of list.getEntries()) { + if ( + entry.entryType === 'paint' || + entry.entryType === 'largest-contentful-paint' + ) { + window.perfMonitor?.measures.set(entry.name, entry.startTime) + } + } + }) + observer.observe({ entryTypes: ['paint', 'largest-contentful-paint'] }) + } + }, testName) + } + + async markEvent(eventName: string) { + await this.page.evaluate((eventName) => { + if (window.perfMonitor) { + const markName = `${window.perfMonitor.testName}-${eventName}` + performance.mark(markName) + window.perfMonitor.marks.set( + eventName, + performance.now() - window.perfMonitor.startTime + ) + } + }, eventName) + } + + async measureOperation( + operationName: string, + operation: () => Promise + ): Promise { + await this.markEvent(`${operationName}-start`) + const result = await operation() + await this.markEvent(`${operationName}-end`) + + // Create performance measure + await this.page.evaluate((operationName) => { + if (window.perfMonitor) { + const testName = window.perfMonitor.testName + const startMark = `${testName}-${operationName}-start` + const endMark = `${testName}-${operationName}-end` + + try { + performance.measure(`${operationName}`, startMark, endMark) + const measure = performance.getEntriesByName(`${operationName}`)[0] + window.perfMonitor.measures.set(operationName, measure.duration) + } catch (e) { + console.warn('Failed to create performance measure:', e) + } + } + }, operationName) + + return result + } + + async collectMetrics( + testName: string, + branch: string = 'unknown' + ): Promise { + const metrics = await this.page.evaluate( + ({ testName, branch }) => { + if (!window.perfMonitor) return null + + // Collect all performance data + const navigationEntry = performance.getEntriesByType( + 'navigation' + )[0] as PerformanceNavigationTiming + const paintEntries = performance.getEntriesByType('paint') + const lcpEntries = performance.getEntriesByType( + 'largest-contentful-paint' + ) + + const timing: any = {} + if (navigationEntry) { + timing.loadStart = navigationEntry.loadEventStart + timing.domContentLoaded = navigationEntry.domContentLoadedEventEnd + timing.loadComplete = navigationEntry.loadEventEnd + } + + paintEntries.forEach((entry) => { + if (entry.name === 'first-paint') { + timing.firstPaint = entry.startTime + } else if (entry.name === 'first-contentful-paint') { + timing.firstContentfulPaint = entry.startTime + } + }) + + if (lcpEntries.length > 0) { + timing.largestContentfulPaint = + lcpEntries[lcpEntries.length - 1].startTime + } + + const customMetrics: Record = {} + window.perfMonitor.measures.forEach((value, key) => { + customMetrics[key] = value + }) + + return { + testName, + timestamp: Date.now(), + branch, + memoryUsage: performance.memory + ? { + usedJSHeapSize: performance.memory.usedJSHeapSize, + totalJSHeapSize: performance.memory.totalJSHeapSize, + jsHeapSizeLimit: performance.memory.jsHeapSizeLimit + } + : null, + timing, + customMetrics + } + }, + { testName, branch } + ) + + if (metrics) { + this.metrics.push(metrics) + console.log('PERFORMANCE_METRICS:', JSON.stringify(metrics)) + } + + return metrics + } + + async finishMonitoring(testName: string) { + await this.markEvent('test-end') + await this.collectMetrics(testName, 'vue-widget/perf-test') + console.log('Finishing performance monitoring') + // Print all metrics + console.log('PERFORMANCE_METRICS:', JSON.stringify(this.metrics)) + + // Cleanup + await this.page.evaluate(() => { + if (window.perfMonitor) { + delete window.perfMonitor + } + }) + } + + getAllMetrics(): PerformanceMetrics[] { + return this.metrics + } +} + +// Extend window type for TypeScript +declare global { + interface Window { + perfMonitor?: { + testName: string + startTime: number + marks: Map + measures: Map + } + } + + // Chrome-specific performance.memory extension + interface Performance { + memory?: { + usedJSHeapSize: number + totalJSHeapSize: number + jsHeapSizeLimit: number + } + } +} diff --git a/browser_tests/tests/copyPaste.spec.ts b/browser_tests/tests/copyPaste.spec.ts index 3bcee65f0..f0628e6b4 100644 --- a/browser_tests/tests/copyPaste.spec.ts +++ b/browser_tests/tests/copyPaste.spec.ts @@ -1,6 +1,7 @@ import { expect } from '@playwright/test' import { comfyPageFixture as test } from '../fixtures/ComfyPage' +import { PerformanceMonitor } from '../helpers/performanceMonitor' test.describe('Copy Paste', () => { test('Can copy and paste node', async ({ comfyPage }) => { @@ -11,107 +12,288 @@ test.describe('Copy Paste', () => { await expect(comfyPage.canvas).toHaveScreenshot('copied-node.png') }) - test('Can copy and paste node with link', async ({ comfyPage }) => { - await comfyPage.clickTextEncodeNode1() - await comfyPage.page.mouse.move(10, 10) - await comfyPage.ctrlC() - await comfyPage.page.keyboard.press('Control+Shift+V') - await expect(comfyPage.canvas).toHaveScreenshot('copied-node-with-link.png') + test('@perf Can copy and paste node with link', async ({ comfyPage }) => { + const perfMonitor = new PerformanceMonitor(comfyPage.page) + const testName = 'copy-paste-node-with-link' + + await perfMonitor.startMonitoring(testName) + + // Click node with performance tracking + await perfMonitor.measureOperation('click-text-encode-node', async () => { + await comfyPage.clickTextEncodeNode1() + }) + + // Mouse move with performance tracking + await perfMonitor.measureOperation('mouse-move', async () => { + await comfyPage.page.mouse.move(10, 10) + }) + + // Copy operation with performance tracking + await perfMonitor.measureOperation('copy-operation', async () => { + await comfyPage.ctrlC() + }) + + // Mark before paste + await perfMonitor.markEvent('before-paste') + + // Paste operation with performance tracking + await perfMonitor.measureOperation('paste-operation', async () => { + await comfyPage.page.keyboard.press('Control+Shift+V') + }) + + // Mark after paste + await perfMonitor.markEvent('after-paste') + + await perfMonitor.finishMonitoring(testName) }) - test('Can copy and paste text', async ({ comfyPage }) => { + test('@perf Can copy and paste text', async ({ comfyPage }) => { + const perfMonitor = new PerformanceMonitor(comfyPage.page) + const testName = 'copy-paste-text' + + await perfMonitor.startMonitoring(testName) + const textBox = comfyPage.widgetTextBox - await textBox.click() - const originalString = await textBox.inputValue() - await textBox.selectText() - await comfyPage.ctrlC(null) - await comfyPage.ctrlV(null) - await comfyPage.ctrlV(null) + + await perfMonitor.measureOperation('click-textbox', async () => { + await textBox.click() + }) + + let originalString: string + await perfMonitor.measureOperation('get-input-value', async () => { + originalString = await textBox.inputValue() + }) + + await perfMonitor.measureOperation('select-text', async () => { + await textBox.selectText() + }) + + await perfMonitor.measureOperation('copy-text', async () => { + await comfyPage.ctrlC(null) + }) + + await perfMonitor.measureOperation('paste-text-first', async () => { + await comfyPage.ctrlV(null) + }) + + await perfMonitor.measureOperation('paste-text-second', async () => { + await comfyPage.ctrlV(null) + }) + const resultString = await textBox.inputValue() - expect(resultString).toBe(originalString + originalString) + expect(resultString).toBe(originalString! + originalString!) + + await perfMonitor.finishMonitoring(testName) }) - test('Can copy and paste widget value', async ({ comfyPage }) => { + test('@perf Can copy and paste widget value', async ({ comfyPage }) => { + const perfMonitor = new PerformanceMonitor(comfyPage.page) + const testName = 'copy-paste-widget-value' + + await perfMonitor.startMonitoring(testName) + // Copy width value (512) from empty latent node to KSampler's seed. // KSampler's seed - await comfyPage.canvas.click({ - position: { - x: 1005, - y: 281 - } + await perfMonitor.measureOperation('click-ksampler-seed', async () => { + await comfyPage.canvas.click({ + position: { + x: 1005, + y: 281 + } + }) }) - await comfyPage.ctrlC(null) + + await perfMonitor.measureOperation('copy-widget-value', async () => { + await comfyPage.ctrlC(null) + }) + // Empty latent node's width - await comfyPage.canvas.click({ - position: { - x: 718, - y: 643 - } + await perfMonitor.measureOperation('click-empty-latent-width', async () => { + await comfyPage.canvas.click({ + position: { + x: 718, + y: 643 + } + }) }) - await comfyPage.ctrlV(null) - await comfyPage.page.keyboard.press('Enter') - await expect(comfyPage.canvas).toHaveScreenshot('copied-widget-value.png') + + await perfMonitor.measureOperation('paste-widget-value', async () => { + await comfyPage.ctrlV(null) + }) + + await perfMonitor.measureOperation('confirm-with-enter', async () => { + await comfyPage.page.keyboard.press('Enter') + }) + + await perfMonitor.finishMonitoring(testName) }) /** * https://github.com/Comfy-Org/ComfyUI_frontend/issues/98 */ - test('Paste in text area with node previously copied', async ({ + test('@perf Paste in text area with node previously copied', async ({ comfyPage }) => { - await comfyPage.clickEmptyLatentNode() - await comfyPage.ctrlC(null) + const perfMonitor = new PerformanceMonitor(comfyPage.page) + const testName = 'paste-text-with-node-copied' + + await perfMonitor.startMonitoring(testName) + + await perfMonitor.measureOperation('click-empty-latent-node', async () => { + await comfyPage.clickEmptyLatentNode() + }) + + await perfMonitor.measureOperation('copy-node', async () => { + await comfyPage.ctrlC(null) + }) + const textBox = comfyPage.widgetTextBox - await textBox.click() - await textBox.inputValue() - await textBox.selectText() - await comfyPage.ctrlC(null) - await comfyPage.ctrlV(null) - await comfyPage.ctrlV(null) - await expect(comfyPage.canvas).toHaveScreenshot( - 'paste-in-text-area-with-node-previously-copied.png' - ) + + await perfMonitor.measureOperation('click-textbox', async () => { + await textBox.click() + }) + + await perfMonitor.measureOperation('get-input-value', async () => { + await textBox.inputValue() + }) + + await perfMonitor.measureOperation('select-text', async () => { + await textBox.selectText() + }) + + await perfMonitor.measureOperation('copy-text', async () => { + await comfyPage.ctrlC(null) + }) + + await perfMonitor.measureOperation('paste-text-first', async () => { + await comfyPage.ctrlV(null) + }) + + await perfMonitor.measureOperation('paste-text-second', async () => { + await comfyPage.ctrlV(null) + }) + + await perfMonitor.finishMonitoring(testName) }) - test('Copy text area does not copy node', async ({ comfyPage }) => { + test('@perf Copy text area does not copy node', async ({ comfyPage }) => { + const perfMonitor = new PerformanceMonitor(comfyPage.page) + const testName = 'copy-text-no-node' + + await perfMonitor.startMonitoring(testName) + const textBox = comfyPage.widgetTextBox - await textBox.click() - await textBox.inputValue() - await textBox.selectText() - await comfyPage.ctrlC(null) + + await perfMonitor.measureOperation('click-textbox', async () => { + await textBox.click() + }) + + await perfMonitor.measureOperation('get-input-value', async () => { + await textBox.inputValue() + }) + + await perfMonitor.measureOperation('select-text', async () => { + await textBox.selectText() + }) + + await perfMonitor.measureOperation('copy-text', async () => { + await comfyPage.ctrlC(null) + }) + // Unfocus textbox. - await comfyPage.page.mouse.click(10, 10) - await comfyPage.ctrlV(null) - await expect(comfyPage.canvas).toHaveScreenshot('no-node-copied.png') + await perfMonitor.measureOperation('unfocus-textbox', async () => { + await comfyPage.page.mouse.click(10, 10) + }) + + await perfMonitor.measureOperation('paste-attempt', async () => { + await comfyPage.ctrlV(null) + }) + + await perfMonitor.finishMonitoring(testName) }) - test('Copy node by dragging + alt', async ({ comfyPage }) => { + test('@perf Copy node by dragging + alt', async ({ comfyPage }) => { + const perfMonitor = new PerformanceMonitor(comfyPage.page) + const testName = 'copy-node-drag-alt' + + await perfMonitor.startMonitoring(testName) + // TextEncodeNode1 - await comfyPage.page.mouse.move(618, 191) + await perfMonitor.measureOperation('mouse-move-to-node', async () => { + await comfyPage.page.mouse.move(618, 191) + }) + + await perfMonitor.markEvent('alt-key-down') await comfyPage.page.keyboard.down('Alt') - await comfyPage.page.mouse.down() - await comfyPage.page.mouse.move(100, 100) - await comfyPage.page.mouse.up() + + await perfMonitor.measureOperation('mouse-down', async () => { + await comfyPage.page.mouse.down() + }) + + await perfMonitor.measureOperation('drag-node', async () => { + await comfyPage.page.mouse.move(100, 100) + }) + + await perfMonitor.measureOperation('mouse-up', async () => { + await comfyPage.page.mouse.up() + }) + + await perfMonitor.markEvent('alt-key-up') await comfyPage.page.keyboard.up('Alt') - await expect(comfyPage.canvas).toHaveScreenshot('drag-copy-copied-node.png') + + await perfMonitor.finishMonitoring(testName) }) - test('Can undo paste multiple nodes as single action', async ({ + test('@perf Can undo paste multiple nodes as single action', async ({ comfyPage }) => { - const initialCount = await comfyPage.getGraphNodesCount() - expect(initialCount).toBeGreaterThan(1) - await comfyPage.canvas.click() - await comfyPage.ctrlA() - await comfyPage.page.mouse.move(10, 10) - await comfyPage.ctrlC() - await comfyPage.ctrlV() + const perfMonitor = new PerformanceMonitor(comfyPage.page) + const testName = 'undo-paste-multiple-nodes' - const pasteCount = await comfyPage.getGraphNodesCount() - expect(pasteCount).toBe(initialCount * 2) + await perfMonitor.startMonitoring(testName) - await comfyPage.ctrlZ() - const undoCount = await comfyPage.getGraphNodesCount() - expect(undoCount).toBe(initialCount) + let initialCount: number + await perfMonitor.measureOperation('get-initial-count', async () => { + initialCount = await comfyPage.getGraphNodesCount() + }) + expect(initialCount!).toBeGreaterThan(1) + + await perfMonitor.measureOperation('click-canvas', async () => { + await comfyPage.canvas.click() + }) + + await perfMonitor.measureOperation('select-all', async () => { + await comfyPage.ctrlA() + }) + + await perfMonitor.measureOperation('mouse-move', async () => { + await comfyPage.page.mouse.move(10, 10) + }) + + await perfMonitor.measureOperation('copy-all-nodes', async () => { + await comfyPage.ctrlC() + }) + + await perfMonitor.measureOperation('paste-all-nodes', async () => { + await comfyPage.ctrlV() + }) + + let pasteCount: number + await perfMonitor.measureOperation('get-paste-count', async () => { + pasteCount = await comfyPage.getGraphNodesCount() + }) + expect(pasteCount!).toBe(initialCount! * 2) + + await perfMonitor.measureOperation('undo-paste', async () => { + await comfyPage.ctrlZ() + }) + + let undoCount: number + await perfMonitor.measureOperation('get-undo-count', async () => { + undoCount = await comfyPage.getGraphNodesCount() + }) + expect(undoCount!).toBe(initialCount!) + + await perfMonitor.finishMonitoring(testName) }) }) diff --git a/perf-test.sh b/perf-test.sh new file mode 100644 index 000000000..9e5477fd6 --- /dev/null +++ b/perf-test.sh @@ -0,0 +1,12 @@ +# Run performance tests with more detailed output +npx playwright test --workers 1 --project=performance --reporter=line + +# Run performance tests on specific files +#npx playwright test --workers 1 --project=performance interaction.spec.ts + +# Run performance tests with trace for debugging +#npx playwright test --workers 1 --project=performance --trace=on + +# Run performance tests and update any snapshots +#npx playwright test --workers 1 --project=performance --update-snapshots + diff --git a/playwright.config.ts b/playwright.config.ts index 0bccb1582..9f74d74c0 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -39,7 +39,7 @@ export default defineConfig({ name: 'chromium', use: { ...devices['Desktop Chrome'] }, timeout: 15000, - grepInvert: /@mobile/ // Run all tests except those tagged with @mobile + grepInvert: /@mobile|@perf/ // Run all tests except those tagged with @mobile or @perf }, { @@ -49,6 +49,21 @@ export default defineConfig({ grep: /@2x/ // Run all tests tagged with @2x }, + { + // Set workers in cli or in upper config + name: 'performance', + use: { + ...devices['Desktop Chrome'], + // Single worker for consistent performance measurements + trace: 'retain-on-failure' + }, + timeout: 30000, // Longer timeout for performance tests + grep: /@perf/, // Run only tests tagged with @perf + ignoreSnapshots: true, + // repeatEach: 5, + fullyParallel: false + }, + // { // name: 'firefox', // use: { ...devices['Desktop Firefox'] },