Restore backend state when Playwright test finishes (#1168)

* Restore backend state when Playwright test finishes. Resolve #1094

* Add warning when env var not set

* Rename and replace with scaffolding option for models dir

* Rename

* Define another env var [skip ci]

* Fix paths [skip ci]

* Update README.md

---------

Co-authored-by: Chenlei Hu <huchenlei@proton.me>

.env_example

@@ -12,6 +12,10 @@ DEV_SERVER_COMFYUI_URL=http://127.0.0.1:8188
 # to ComfyUI launch script to serve the custom web version.
 DEPLOY_COMFYUI_DIR=/home/ComfyUI/web
 
+# The directory containing the ComfyUI installation used to run Playwright tests.
+# If you aren't using a separate install for testing, point this to your regular install.
+TEST_COMFYUI_DIR=/home/ComfyUI
+
 # The directory containing the ComfyUI_examples repo used to extract test workflows.
 EXAMPLE_REPO_PATH=tests-ui/ComfyUI_examples
 

browser_tests/README.md

@@ -5,7 +5,7 @@ This document outlines the setup and usage of Playwright for testing the ComfyUI
 ## WARNING
 
 The browser tests will change the ComfyUI backend state, such as user settings and saved workflows.
-Please backup your ComfyUI data before running the tests locally.
+If `TEST_COMFYUI_DIR` in `.env` isn't set to your `(Comfy Path)/ComfyUI` directory, these changes won't be automatically restored.
 
 ## Setup
 

browser_tests/globalSetup.ts

@@ -0,0 +1,20 @@
+import { FullConfig } from '@playwright/test'
+import { backupPath } from './utils/backupUtils'
+import dotenv from 'dotenv'
+
+dotenv.config()
+
+export default function globalSetup(config: FullConfig) {
+  if (!process.env.CI) {
+    if (process.env.TEST_COMFYUI_DIR) {
+      backupPath([process.env.TEST_COMFYUI_DIR, 'user'])
+      backupPath([process.env.TEST_COMFYUI_DIR, 'models'], {
+        renameAndReplaceWithScaffolding: true
+      })
+    } else {
+      console.warn(
+        'Set TEST_COMFYUI_DIR in .env to prevent user data (settings, workflows, etc.) from being overwritten'
+      )
+    }
+  }
+}

browser_tests/globalTeardown.ts

@@ -0,0 +1,12 @@
+import { FullConfig } from '@playwright/test'
+import { restorePath } from './utils/backupUtils'
+import dotenv from 'dotenv'
+
+dotenv.config()
+
+export default function globalTeardown(config: FullConfig) {
+  if (!process.env.CI && process.env.TEST_COMFYUI_DIR) {
+    restorePath([process.env.TEST_COMFYUI_DIR, 'user'])
+    restorePath([process.env.TEST_COMFYUI_DIR, 'models'])
+  }
+}

browser_tests/utils/backupUtils.ts

@@ -0,0 +1,69 @@
+import path from 'path'
+import fs from 'fs-extra'
+
+type PathParts = readonly [string, ...string[]]
+
+const getBackupPath = (originalPath: string): string => `${originalPath}.bak`
+
+const resolvePathIfExists = (pathParts: PathParts): string | null => {
+  const resolvedPath = path.resolve(...pathParts)
+  if (!fs.pathExistsSync(resolvedPath)) {
+    console.warn(`Path not found: ${resolvedPath}`)
+    return null
+  }
+  return resolvedPath
+}
+
+const createScaffoldingCopy = (srcDir: string, destDir: string) => {
+  // Get all items (files and directories) in the source directory
+  const items = fs.readdirSync(srcDir, { withFileTypes: true })
+
+  for (const item of items) {
+    const srcPath = path.join(srcDir, item.name)
+    const destPath = path.join(destDir, item.name)
+
+    if (item.isDirectory()) {
+      // Create the corresponding directory in the destination
+      fs.ensureDirSync(destPath)
+
+      // Recursively copy the directory structure
+      createScaffoldingCopy(srcPath, destPath)
+    }
+  }
+}
+
+export function backupPath(
+  pathParts: PathParts,
+  { renameAndReplaceWithScaffolding = false } = {}
+) {
+  const originalPath = resolvePathIfExists(pathParts)
+  if (!originalPath) return
+
+  const backupPath = getBackupPath(originalPath)
+  try {
+    if (renameAndReplaceWithScaffolding) {
+      // Rename the original path and create scaffolding in its place
+      fs.moveSync(originalPath, backupPath)
+      createScaffoldingCopy(backupPath, originalPath)
+    } else {
+      // Create a copy of the original path
+      fs.copySync(originalPath, backupPath)
+    }
+  } catch (error) {
+    console.error(`Failed to backup ${originalPath} from ${backupPath}`, error)
+  }
+}
+
+export function restorePath(pathParts: PathParts) {
+  const originalPath = resolvePathIfExists(pathParts)
+  if (!originalPath) return
+
+  const backupPath = getBackupPath(originalPath)
+  if (!fs.pathExistsSync(backupPath)) return
+
+  try {
+    fs.moveSync(backupPath, originalPath, { overwrite: true })
+  } catch (error) {
+    console.error(`Failed to restore ${originalPath} from ${backupPath}`, error)
+  }
+}

playwright.config.ts

@@ -30,6 +30,10 @@ export default defineConfig({
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
     trace: 'on-first-retry'
   },
+  /* Path to global setup file. Exported function runs once before all the tests */
+  globalSetup: './browser_tests/globalSetup.ts',
+  /* Path to global teardown file. Exported function runs once after all the tests */
+  globalTeardown: './browser_tests/globalTeardown.ts',
 
   /* Configure projects for major browsers */
   projects: [