From fabcbaec8260b58dc0ba4324beb7815a5a56d664 Mon Sep 17 00:00:00 2001 From: bymyself Date: Wed, 9 Oct 2024 12:05:19 -0700 Subject: [PATCH] Restore backend state when Playwright test finishes (#1168) * Restore backend state when Playwright test finishes. Resolve #1094 * Add warning when env var not set * Rename and replace with scaffolding option for models dir * Rename * Define another env var [skip ci] * Fix paths [skip ci] * Update README.md --------- Co-authored-by: Chenlei Hu --- .env_example | 4 ++ browser_tests/README.md | 2 +- browser_tests/globalSetup.ts | 20 +++++++++ browser_tests/globalTeardown.ts | 12 ++++++ browser_tests/utils/backupUtils.ts | 69 ++++++++++++++++++++++++++++++ playwright.config.ts | 4 ++ 6 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 browser_tests/globalSetup.ts create mode 100644 browser_tests/globalTeardown.ts create mode 100644 browser_tests/utils/backupUtils.ts diff --git a/.env_example b/.env_example index 3ddf31fa2..7bebc6936 100644 --- a/.env_example +++ b/.env_example @@ -12,6 +12,10 @@ DEV_SERVER_COMFYUI_URL=http://127.0.0.1:8188 # to ComfyUI launch script to serve the custom web version. DEPLOY_COMFYUI_DIR=/home/ComfyUI/web +# The directory containing the ComfyUI installation used to run Playwright tests. +# If you aren't using a separate install for testing, point this to your regular install. +TEST_COMFYUI_DIR=/home/ComfyUI + # The directory containing the ComfyUI_examples repo used to extract test workflows. EXAMPLE_REPO_PATH=tests-ui/ComfyUI_examples diff --git a/browser_tests/README.md b/browser_tests/README.md index 1a58c8988..b63db0ddb 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -5,7 +5,7 @@ This document outlines the setup and usage of Playwright for testing the ComfyUI ## WARNING The browser tests will change the ComfyUI backend state, such as user settings and saved workflows. -Please backup your ComfyUI data before running the tests locally. +If `TEST_COMFYUI_DIR` in `.env` isn't set to your `(Comfy Path)/ComfyUI` directory, these changes won't be automatically restored. ## Setup diff --git a/browser_tests/globalSetup.ts b/browser_tests/globalSetup.ts new file mode 100644 index 000000000..86626480f --- /dev/null +++ b/browser_tests/globalSetup.ts @@ -0,0 +1,20 @@ +import { FullConfig } from '@playwright/test' +import { backupPath } from './utils/backupUtils' +import dotenv from 'dotenv' + +dotenv.config() + +export default function globalSetup(config: FullConfig) { + if (!process.env.CI) { + if (process.env.TEST_COMFYUI_DIR) { + backupPath([process.env.TEST_COMFYUI_DIR, 'user']) + backupPath([process.env.TEST_COMFYUI_DIR, 'models'], { + renameAndReplaceWithScaffolding: true + }) + } else { + console.warn( + 'Set TEST_COMFYUI_DIR in .env to prevent user data (settings, workflows, etc.) from being overwritten' + ) + } + } +} diff --git a/browser_tests/globalTeardown.ts b/browser_tests/globalTeardown.ts new file mode 100644 index 000000000..8ebb71339 --- /dev/null +++ b/browser_tests/globalTeardown.ts @@ -0,0 +1,12 @@ +import { FullConfig } from '@playwright/test' +import { restorePath } from './utils/backupUtils' +import dotenv from 'dotenv' + +dotenv.config() + +export default function globalTeardown(config: FullConfig) { + if (!process.env.CI && process.env.TEST_COMFYUI_DIR) { + restorePath([process.env.TEST_COMFYUI_DIR, 'user']) + restorePath([process.env.TEST_COMFYUI_DIR, 'models']) + } +} diff --git a/browser_tests/utils/backupUtils.ts b/browser_tests/utils/backupUtils.ts new file mode 100644 index 000000000..0d2ae1f4d --- /dev/null +++ b/browser_tests/utils/backupUtils.ts @@ -0,0 +1,69 @@ +import path from 'path' +import fs from 'fs-extra' + +type PathParts = readonly [string, ...string[]] + +const getBackupPath = (originalPath: string): string => `${originalPath}.bak` + +const resolvePathIfExists = (pathParts: PathParts): string | null => { + const resolvedPath = path.resolve(...pathParts) + if (!fs.pathExistsSync(resolvedPath)) { + console.warn(`Path not found: ${resolvedPath}`) + return null + } + return resolvedPath +} + +const createScaffoldingCopy = (srcDir: string, destDir: string) => { + // Get all items (files and directories) in the source directory + const items = fs.readdirSync(srcDir, { withFileTypes: true }) + + for (const item of items) { + const srcPath = path.join(srcDir, item.name) + const destPath = path.join(destDir, item.name) + + if (item.isDirectory()) { + // Create the corresponding directory in the destination + fs.ensureDirSync(destPath) + + // Recursively copy the directory structure + createScaffoldingCopy(srcPath, destPath) + } + } +} + +export function backupPath( + pathParts: PathParts, + { renameAndReplaceWithScaffolding = false } = {} +) { + const originalPath = resolvePathIfExists(pathParts) + if (!originalPath) return + + const backupPath = getBackupPath(originalPath) + try { + if (renameAndReplaceWithScaffolding) { + // Rename the original path and create scaffolding in its place + fs.moveSync(originalPath, backupPath) + createScaffoldingCopy(backupPath, originalPath) + } else { + // Create a copy of the original path + fs.copySync(originalPath, backupPath) + } + } catch (error) { + console.error(`Failed to backup ${originalPath} from ${backupPath}`, error) + } +} + +export function restorePath(pathParts: PathParts) { + const originalPath = resolvePathIfExists(pathParts) + if (!originalPath) return + + const backupPath = getBackupPath(originalPath) + if (!fs.pathExistsSync(backupPath)) return + + try { + fs.moveSync(backupPath, originalPath, { overwrite: true }) + } catch (error) { + console.error(`Failed to restore ${originalPath} from ${backupPath}`, error) + } +} diff --git a/playwright.config.ts b/playwright.config.ts index 45807ad82..a915a209d 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -30,6 +30,10 @@ export default defineConfig({ /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: 'on-first-retry' }, + /* Path to global setup file. Exported function runs once before all the tests */ + globalSetup: './browser_tests/globalSetup.ts', + /* Path to global teardown file. Exported function runs once after all the tests */ + globalTeardown: './browser_tests/globalTeardown.ts', /* Configure projects for major browsers */ projects: [