diff --git a/.gitignore b/.gitignore index 3e400ba39..6592721b7 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,7 @@ coverage/ /playwright/.cache/ browser_tests/**/*-win32.png browser_tests/local/ +browser_tests/.auth/ .env diff --git a/browser_tests/fixtures/CloudComfyPage.ts b/browser_tests/fixtures/CloudComfyPage.ts new file mode 100644 index 000000000..8df51081f --- /dev/null +++ b/browser_tests/fixtures/CloudComfyPage.ts @@ -0,0 +1,40 @@ +import type { APIRequestContext, Page } from '@playwright/test' + +import { ComfyPage } from './ComfyPage' +import type { FolderStructure } from './ComfyPage' + +/** + * Cloud-specific implementation of ComfyPage. + * Uses Firebase auth persistence and cloud API for settings. + */ +export class CloudComfyPage extends ComfyPage { + constructor(page: Page, request: APIRequestContext) { + super(page, request) + } + + async setupUser(username: string): Promise { + // No-op for cloud - user already authenticated via Firebase in globalSetup + // Firebase auth is persisted via storageState in the fixture + return null + } + + async setupSettings(settings: Record): Promise { + // Cloud uses batch settings API (not devtools) + // Firebase auth token is automatically included from restored localStorage + const resp = await this.request.post(`${this.url}/api/settings`, { + data: settings + }) + + if (!resp.ok()) { + throw new Error(`Failed to setup cloud settings: ${await resp.text()}`) + } + } + + async setupWorkflowsDirectory(structure: FolderStructure): Promise { + // Cloud workflow API not yet implemented + // For initial smoke tests, we can skip this functionality + console.warn( + 'setupWorkflowsDirectory: not yet implemented for cloud mode - skipping' + ) + } +} diff --git a/browser_tests/fixtures/ComfyPage.ts b/browser_tests/fixtures/ComfyPage.ts index 8059fb4cb..5c5e3364d 100644 --- a/browser_tests/fixtures/ComfyPage.ts +++ b/browser_tests/fixtures/ComfyPage.ts @@ -11,6 +11,7 @@ import { NodeBadgeMode } from '../../src/types/nodeSource' import { ComfyActionbar } from '../helpers/actionbar' import { ComfyTemplates } from '../helpers/templates' import { ComfyMouse } from './ComfyMouse' +import { LocalhostComfyPage } from './LocalhostComfyPage' import { VueNodeHelpers } from './VueNodeHelpers' import { ComfyNodeSearchBox } from './components/ComfyNodeSearchBox' import { SettingDialog } from './components/SettingDialog' @@ -94,7 +95,7 @@ class ComfyMenu { } } -type FolderStructure = { +export type FolderStructure = { [key: string]: FolderStructure | string } @@ -122,7 +123,11 @@ class ConfirmDialog { } } -export class ComfyPage { +/** + * Abstract base class for ComfyUI page objects. + * Subclasses must implement backend-specific methods for different environments (localhost, cloud, etc.) + */ +export abstract class ComfyPage { private _history: TaskHistory | null = null public readonly url: string @@ -215,65 +220,24 @@ export class ComfyPage { }) } - async setupWorkflowsDirectory(structure: FolderStructure) { - const resp = await this.request.post( - `${this.url}/api/devtools/setup_folder_structure`, - { - data: { - tree_structure: this.convertLeafToContent(structure), - base_path: `user/${this.id}/workflows` - } - } - ) + /** + * Setup workflows directory structure. Implementation varies by environment. + * @param structure - Folder structure to create + */ + abstract setupWorkflowsDirectory(structure: FolderStructure): Promise - if (resp.status() !== 200) { - throw new Error( - `Failed to setup workflows directory: ${await resp.text()}` - ) - } + /** + * Setup user for testing. Implementation varies by environment. + * @param username - Username to setup + * @returns User ID or null if not applicable + */ + abstract setupUser(username: string): Promise - await this.page.evaluate(async () => { - await window['app'].extensionManager.workflow.syncWorkflows() - }) - } - - async setupUser(username: string) { - const res = await this.request.get(`${this.url}/api/users`) - if (res.status() !== 200) - throw new Error(`Failed to retrieve users: ${await res.text()}`) - - const apiRes = await res.json() - const user = Object.entries(apiRes?.users ?? {}).find( - ([, name]) => name === username - ) - const id = user?.[0] - - return id ? id : await this.createUser(username) - } - - async createUser(username: string) { - const resp = await this.request.post(`${this.url}/api/users`, { - data: { username } - }) - - if (resp.status() !== 200) - throw new Error(`Failed to create user: ${await resp.text()}`) - - return await resp.json() - } - - async setupSettings(settings: Record) { - const resp = await this.request.post( - `${this.url}/api/devtools/set_settings`, - { - data: settings - } - ) - - if (resp.status() !== 200) { - throw new Error(`Failed to setup settings: ${await resp.text()}`) - } - } + /** + * Setup settings for testing. Implementation varies by environment. + * @param settings - Settings object to apply + */ + abstract setupSettings(settings: Record): Promise setupHistory(): TaskHistory { this._history ??= new TaskHistory(this) @@ -1635,12 +1599,14 @@ export const comfyPageFixture = base.extend<{ comfyMouse: ComfyMouse }>({ comfyPage: async ({ page, request }, use, testInfo) => { - const comfyPage = new ComfyPage(page, request) + const comfyPage = new LocalhostComfyPage(page, request) const { parallelIndex } = testInfo const username = `playwright-test-${parallelIndex}` const userId = await comfyPage.setupUser(username) - comfyPage.userIds[parallelIndex] = userId + if (userId) { + comfyPage.userIds[parallelIndex] = userId + } try { await comfyPage.setupSettings({ diff --git a/browser_tests/fixtures/ComfyPageCloud.ts b/browser_tests/fixtures/ComfyPageCloud.ts new file mode 100644 index 000000000..cce53ae0d --- /dev/null +++ b/browser_tests/fixtures/ComfyPageCloud.ts @@ -0,0 +1,48 @@ +import { test as base } from '@playwright/test' + +import { CloudComfyPage } from './CloudComfyPage' +import { ComfyMouse } from './ComfyMouse' +import type { ComfyPage } from './ComfyPage' + +/** + * Cloud-specific fixture for ComfyPage. + * Uses Firebase auth persisted from globalSetupCloud.ts. + */ +export const comfyPageCloudFixture = base.extend<{ + comfyPage: ComfyPage + comfyMouse: ComfyMouse +}>({ + // Use the storageState saved by globalSetupCloud + storageState: 'browser_tests/.auth/cloudUser.json', + + comfyPage: async ({ page, request }, use) => { + const comfyPage = new CloudComfyPage(page, request) + + // Note: No setupUser needed - Firebase auth persisted via storageState + // Setup cloud-specific settings (optional - can customize per test) + try { + await comfyPage.setupSettings({ + 'Comfy.UseNewMenu': 'Top', + // Hide canvas menu/info/selection toolbox by default. + 'Comfy.Graph.CanvasInfo': false, + 'Comfy.Graph.CanvasMenu': false, + 'Comfy.Canvas.SelectionToolbox': false, + // Disable tooltips by default to avoid flakiness. + 'Comfy.EnableTooltips': false, + // Set tutorial completed to true to avoid loading the tutorial workflow. + 'Comfy.TutorialCompleted': true + }) + } catch (e) { + console.error('Failed to setup cloud settings:', e) + } + + // Don't mock releases for cloud - cloud handles its own releases + await comfyPage.setup({ mockReleases: false }) + await use(comfyPage) + }, + + comfyMouse: async ({ comfyPage }, use) => { + const comfyMouse = new ComfyMouse(comfyPage) + await use(comfyMouse) + } +}) diff --git a/browser_tests/fixtures/LocalhostComfyPage.ts b/browser_tests/fixtures/LocalhostComfyPage.ts new file mode 100644 index 000000000..30807ca9e --- /dev/null +++ b/browser_tests/fixtures/LocalhostComfyPage.ts @@ -0,0 +1,74 @@ +import type { APIRequestContext, Page } from '@playwright/test' + +import { ComfyPage } from './ComfyPage' +import type { FolderStructure } from './ComfyPage' + +/** + * Localhost-specific implementation of ComfyPage. + * Uses devtools API and multi-user mode for test isolation. + */ +export class LocalhostComfyPage extends ComfyPage { + constructor(page: Page, request: APIRequestContext) { + super(page, request) + } + + async setupWorkflowsDirectory(structure: FolderStructure): Promise { + const resp = await this.request.post( + `${this.url}/api/devtools/setup_folder_structure`, + { + data: { + tree_structure: this.convertLeafToContent(structure), + base_path: `user/${this.id}/workflows` + } + } + ) + + if (resp.status() !== 200) { + throw new Error( + `Failed to setup workflows directory: ${await resp.text()}` + ) + } + + await this.page.evaluate(async () => { + await window['app'].extensionManager.workflow.syncWorkflows() + }) + } + + async setupUser(username: string): Promise { + const res = await this.request.get(`${this.url}/api/users`) + if (res.status() !== 200) + throw new Error(`Failed to retrieve users: ${await res.text()}`) + + const apiRes = await res.json() + const user = Object.entries(apiRes?.users ?? {}).find( + ([, name]) => name === username + ) + const id = user?.[0] + + return id ? id : await this.createUser(username) + } + + private async createUser(username: string): Promise { + const resp = await this.request.post(`${this.url}/api/users`, { + data: { username } + }) + + if (resp.status() !== 200) + throw new Error(`Failed to create user: ${await resp.text()}`) + + return await resp.json() + } + + async setupSettings(settings: Record): Promise { + const resp = await this.request.post( + `${this.url}/api/devtools/set_settings`, + { + data: settings + } + ) + + if (resp.status() !== 200) { + throw new Error(`Failed to setup settings: ${await resp.text()}`) + } + } +} diff --git a/browser_tests/globalSetupCloud.ts b/browser_tests/globalSetupCloud.ts new file mode 100644 index 000000000..02ff6be23 --- /dev/null +++ b/browser_tests/globalSetupCloud.ts @@ -0,0 +1,66 @@ +import { chromium } from '@playwright/test' +import type { FullConfig } from '@playwright/test' +import dotenv from 'dotenv' +import * as fs from 'fs' +import * as path from 'path' + +dotenv.config() + +/** + * Global setup for cloud tests. + * Authenticates with Firebase and saves auth state for test reuse. + */ +export default async function globalSetupCloud(config: FullConfig) { + const CLOUD_TEST_EMAIL = process.env.CLOUD_TEST_EMAIL + const CLOUD_TEST_PASSWORD = process.env.CLOUD_TEST_PASSWORD + + if (!CLOUD_TEST_EMAIL || !CLOUD_TEST_PASSWORD) { + throw new Error( + 'CLOUD_TEST_EMAIL and CLOUD_TEST_PASSWORD must be set in environment variables' + ) + } + + const browser = await chromium.launch() + const context = await browser.newContext() + const page = await context.newPage() + + try { + // Navigate to cloud login page + await page.goto('https://stagingcloud.comfy.org/cloud/login', { + waitUntil: 'networkidle', + timeout: 30000 + }) + + // Fill in email and password + await page.fill('input[type="email"]', CLOUD_TEST_EMAIL) + await page.fill('input[type="password"]', CLOUD_TEST_PASSWORD) + + // Click login button + await page.click('button[type="submit"]') + + // Wait for redirect to main app (adjust selector as needed) + await page.waitForURL('**/', { timeout: 30000 }) + + // Wait for app to be fully loaded + await page.waitForFunction( + () => window['app'] && window['app'].extensionManager, + { timeout: 30000 } + ) + + // Ensure .auth directory exists + const authDir = path.join(__dirname, '.auth') + if (!fs.existsSync(authDir)) { + fs.mkdirSync(authDir, { recursive: true }) + } + + // Save authentication state (includes localStorage with Firebase tokens) + await context.storageState({ + path: 'browser_tests/.auth/cloudUser.json' + }) + } catch (error) { + console.error('❌ Failed to authenticate:', error) + throw error + } finally { + await browser.close() + } +} diff --git a/browser_tests/tests/cloud.spec.ts b/browser_tests/tests/cloud.spec.ts new file mode 100644 index 000000000..88b833d1a --- /dev/null +++ b/browser_tests/tests/cloud.spec.ts @@ -0,0 +1,49 @@ +/** + * @cloud + * Cloud E2E tests. + * Tests run against stagingcloud.comfy.org with authenticated user. + */ + +import { expect } from '@playwright/test' + +import { comfyPageCloudFixture as test } from '../fixtures/ComfyPageCloud' + +test.describe('Cloud E2E @cloud', () => { + test('loads app with authentication', async ({ comfyPage }) => { + // App should be loaded from setup() + await expect(comfyPage.canvas).toBeVisible() + + // Verify we're authenticated (cloud-specific check) + const isAuthenticated = await comfyPage.page.evaluate(() => { + // Check for Firebase auth in localStorage + const keys = Object.keys(localStorage) + return keys.some( + (key) => key.startsWith('firebase:') || key.includes('authUser') + ) + }) + expect(isAuthenticated).toBe(true) + }) + + test('can interact with canvas', async ({ comfyPage }) => { + // Basic canvas interaction + await comfyPage.doubleClickCanvas() + await expect(comfyPage.searchBox.input).toBeVisible() + + // Close search box + await comfyPage.page.keyboard.press('Escape') + await expect(comfyPage.searchBox.input).not.toBeVisible() + }) + + test('can access settings dialog', async ({ comfyPage }) => { + // Open settings dialog + await comfyPage.page.click('button[data-testid="settings-button"]', { + timeout: 10000 + }) + + // Settings dialog should be visible + await expect(comfyPage.page.locator('.p-dialog')).toBeVisible() + + // Close settings + await comfyPage.closeDialog() + }) +}) diff --git a/playwright.cloud.config.ts b/playwright.cloud.config.ts new file mode 100644 index 000000000..643c7a32c --- /dev/null +++ b/playwright.cloud.config.ts @@ -0,0 +1,34 @@ +import { defineConfig, devices } from '@playwright/test' + +/** + * Playwright configuration for cloud E2E tests. + * Tests run against stagingcloud.comfy.org with authenticated user. + */ +export default defineConfig({ + testDir: './browser_tests/tests', + fullyParallel: true, + forbidOnly: !!process.env.CI, + reporter: 'html', + + // Cloud tests need more time due to network latency + timeout: 75000, // 5x the default 15000ms + retries: process.env.CI ? 2 : 0, + + use: { + trace: 'on-first-retry', + // Cloud URL - can override with PLAYWRIGHT_TEST_URL env var + baseURL: process.env.PLAYWRIGHT_TEST_URL || 'https://stagingcloud.comfy.org' + }, + + // Authenticate once before all tests + globalSetup: './browser_tests/globalSetupCloud.ts', + + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + timeout: 75000, + grep: /@cloud/ // Only run tests tagged with @cloud + } + ] +}) diff --git a/tsconfig.json b/tsconfig.json index 8319194da..eeb5660a0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -55,6 +55,7 @@ "eslint.config.ts", "global.d.ts", "knip.config.ts", + "playwright.cloud.config.ts", "src/**/*.vue", "src/**/*", "src/types/**/*.d.ts",