Add playwright browser test (#48)

* Add playwright * Add test:browser command * Remove test examples * Add basic node tests * Add drag node test * Merge workflows * nit * Localize jest * Add local config * Change working dir * Use consistent fonts * Fix emoji fonts * Update github action to save expectation * update on test failure * push to head * Update test expectations [skip ci] --------- Co-authored-by: github-actions <github-actions@github.com>
2026-01-26 19:09:52 +00:00 · 2024-06-24 09:41:40 -04:00
parent 3ace859106
commit 51b925f7ef
13 changed files with 370 additions and 8 deletions
--- a/.github/workflows/test-ui.yaml
+++ b/.github/workflows/test-ui.yaml
@@ -1,8 +1,10 @@
 name: Tests CI

 on:
-  - push
-  - pull_request
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]

 jobs:
  test:
@@ -19,9 +21,19 @@ jobs:
      with:
        repository: "huchenlei/ComfyUI_frontend"
        path: "ComfyUI_frontend"
+        ref: ${{ github.head_ref }}
+    - name: Get commit message
+      id: commit-message
+      run: echo "::set-output name=message::$(git log -1 --pretty=%B)"
+      working-directory: ComfyUI_frontend
+    - name: Skip CI
+      if: contains(steps.commit-message.outputs.message, '[skip ci]')
+      run: echo "Skipping CI as commit contains '[skip ci]'"
+      continue-on-error: true
+      working-directory: ComfyUI_frontend
    - uses: actions/setup-node@v3
      with:
-        node-version: 18
+        node-version: lts/*
    - uses: actions/setup-python@v4
      with:
        python-version: '3.10'
@@ -32,14 +44,49 @@ jobs:
        pip install -r requirements.txt
        pip install wait-for-it
      working-directory: ComfyUI
+    - name: Build & Install ComfyUI_frontend
+      run: |
+        npm ci
+        npm run build
+        rm -rf ../ComfyUI/web/*
+        mv dist/* ../ComfyUI/web/
+      working-directory: ComfyUI_frontend
    - name: Start ComfyUI server
      run: |
        python main.py --cpu &
+        wait-for-it --service 127.0.0.1:8188 -t 600
      working-directory: ComfyUI
    - name: Run UI tests
      run: |
-        wait-for-it --service 127.0.0.1:8188 -t 600
-        npm ci
        npm run test:generate
        npm test -- --verbose
      working-directory: ComfyUI_frontend
+    - name: Install Playwright Browsers
+      run: npx playwright install --with-deps
+      working-directory: ComfyUI_frontend
+    - name: Run Playwright tests
+      id: playwright-tests
+      run: npx playwright test
+      continue-on-error: true
+      working-directory: ComfyUI_frontend
+    - uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: playwright-report
+        path: ComfyUI_frontend/playwright-report/
+        retention-days: 30
+    - name: Debugging info
+      run: |
+        echo "Branch: ${{ github.head_ref }}"
+        git status
+      working-directory: ComfyUI_frontend
+    - name: Commit updated expectations
+      if: steps.playwright-tests.outcome == 'failure' && contains(github.event.pull_request.labels.*.name, 'New Browser Test Expectations')
+      # Pushes back to the source branch of the PR
+      run: |
+        git config --global user.name 'github-actions'
+        git config --global user.email 'github-actions@github.com'
+        git add browser_tests
+        git commit -m "Update test expectations [skip ci]"
+        git push origin HEAD:${{ github.head_ref }}
+      working-directory: ComfyUI_frontend
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,11 @@ dist-ssr

 # Ignore test data.
 tests-ui/data/*
+
+# Browser tests
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+
+.env
--- a/browser_tests/ComfyPage.ts
+++ b/browser_tests/ComfyPage.ts
@@ -0,0 +1,88 @@
+import type { Page, Locator } from '@playwright/test';
+import dotenv from "dotenv";
+dotenv.config();
+
+interface Position {
+  x: number;
+  y: number;
+}
+
+export class ComfyPage {
+  public readonly url: string;
+  // All canvas position operations are based on default view of canvas.
+  public readonly canvas: Locator;
+  public readonly widgetTextBox: Locator;
+
+  // Buttons
+  public readonly resetViewButton: Locator;
+
+  constructor(
+    public readonly page: Page,
+  ) {
+    this.url = process.env.PLAYWRIGHT_TEST_URL || 'http://localhost:8188';
+    this.canvas = page.locator('#graph-canvas');
+    this.widgetTextBox = page.getByPlaceholder('text').nth(1);
+    this.resetViewButton = page.getByRole('button', { name: 'Reset View' });
+  }
+
+  async goto() {
+    await this.page.goto(this.url);
+  }
+
+  async nextFrame() {
+    await this.page.evaluate(() => {
+      return new Promise<number>(requestAnimationFrame);
+    });
+  }
+
+  async resetView() {
+    await this.resetViewButton.click();
+    await this.nextFrame();
+  }
+
+  async clickTextEncodeNode1() {
+    await this.canvas.click({
+      position: {
+        x: 618,
+        y: 191
+      }
+    });
+    await this.nextFrame();
+  }
+
+  async clickTextEncodeNode2() {
+    await this.canvas.click({
+      position: {
+        x: 622,
+        y: 400
+      }
+    });
+    await this.nextFrame();
+  }
+
+  async clickEmptySpace() {
+    await this.canvas.click({
+      position: {
+        x: 35,
+        y: 31
+      }
+    });
+    await this.nextFrame();
+  }
+
+  async dragAndDrop(source: Position, target: Position) {
+    await this.page.mouse.move(source.x, source.y);
+    await this.page.mouse.down();
+    await this.page.mouse.move(target.x, target.y);
+    await this.page.mouse.up();
+    await this.nextFrame();
+  }
+
+  async dragNode2() {
+    await this.dragAndDrop(
+      { x: 622, y: 400 },
+      { x: 622, y: 300 },
+    );
+    await this.nextFrame();
+  }
+}
--- a/browser_tests/interaction.spec.ts
+++ b/browser_tests/interaction.spec.ts
@@ -0,0 +1,55 @@
+import { test as base, expect } from '@playwright/test';
+import { ComfyPage } from './ComfyPage';
+
+const test = base.extend<{ comfyPage: ComfyPage }>({
+  comfyPage: async ({ page }, use) => {
+    const comfyPage = new ComfyPage(page);
+    await comfyPage.goto();
+    // Unify font for consistent screenshots.
+    await page.addStyleTag({
+      url: "https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap"
+    });
+    await page.addStyleTag({
+      url: "https://fonts.googleapis.com/css2?family=Noto+Color+Emoji&family=Roboto+Mono:ital,wght@0,100..700;1,100..700&family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap"
+    });
+    await page.addStyleTag({
+      content: `
+      * {
+				font-family: 'Roboto Mono', 'Noto Color Emoji';
+			}`
+    });
+
+    await page.waitForFunction(() => document.fonts.ready);
+    await page.waitForFunction(() => window['app'] != undefined);
+    await page.evaluate(() => { window['app']['canvas'].show_info = false; });
+    await comfyPage.nextFrame();
+    // Reset view to force re-rendering of canvas. So that info fields like fps
+    // become hidden.
+    await comfyPage.resetView();
+    await use(comfyPage);
+  },
+});
+
+test.describe('Node Interaction', () => {
+  test('Can enter prompt', async ({ comfyPage }) => {
+    const textBox = comfyPage.widgetTextBox;
+    await textBox.click();
+    await textBox.fill('Hello World');
+    await expect(textBox).toHaveValue('Hello World');
+    await textBox.fill('Hello World 2');
+    await expect(textBox).toHaveValue('Hello World 2');
+  });
+
+  test('Can highlight selected', async ({ comfyPage }) => {
+    await expect(comfyPage.canvas).toHaveScreenshot('deselected-node.png');
+    await comfyPage.clickTextEncodeNode1();
+    await expect(comfyPage.canvas).toHaveScreenshot('selected-node1.png');
+    await comfyPage.clickTextEncodeNode2();
+    await expect(comfyPage.canvas).toHaveScreenshot('selected-node2.png');
+  });
+
+  test('Can drag node', async ({ comfyPage }) => {
+    await comfyPage.dragNode2();
+    await expect(comfyPage.canvas).toHaveScreenshot('dragged-node1.png');
+  });
+});
--- a/browser_tests/interaction.spec.ts-snapshots/deselected-node-chromium-linux.png
+++ b/browser_tests/interaction.spec.ts-snapshots/deselected-node-chromium-linux.png
--- a/browser_tests/interaction.spec.ts-snapshots/dragged-node1-chromium-linux.png
+++ b/browser_tests/interaction.spec.ts-snapshots/dragged-node1-chromium-linux.png
--- a/browser_tests/interaction.spec.ts-snapshots/selected-node1-chromium-linux.png
+++ b/browser_tests/interaction.spec.ts-snapshots/selected-node1-chromium-linux.png
--- a/browser_tests/interaction.spec.ts-snapshots/selected-node2-chromium-linux.png
+++ b/browser_tests/interaction.spec.ts-snapshots/selected-node2-chromium-linux.png
--- a/index.html
+++ b/index.html
@@ -4,6 +4,14 @@
 		<meta charset="UTF-8">
 		<title>ComfyUI</title>
 		<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
+		<!-- Browser Test Fonts -->
+		<!-- <link href="https://fonts.googleapis.com/css2?family=Roboto+Mono:ital,wght@0,100..700;1,100..700&family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap" rel="stylesheet">
+		<link href="https://fonts.googleapis.com/css2?family=Noto+Color+Emoji&family=Roboto+Mono:ital,wght@0,100..700;1,100..700&family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap" rel="stylesheet">
+		<style>
+			* {
+				font-family: 'Roboto Mono', 'Noto Color Emoji';
+			}
+		</style> -->
 		<link rel="stylesheet" type="text/css" href="/lib/litegraph.css" />
 		<link rel="stylesheet" type="text/css" href="/style.css" />
 		<link rel="stylesheet" type="text/css" href="/user.css" />
--- a/jest.config.ts
+++ b/jest.config.ts
@@ -1,6 +1,7 @@
 import type { JestConfigWithTsJest } from "ts-jest";

 const jestConfig: JestConfigWithTsJest = {
+	testMatch: ["**/tests-ui/**/*.test.ts"],
 	testEnvironment: "jsdom",
 	transform: {
 		'^.+\\.m?[tj]sx?$': ["ts-jest", {
--- a/package-lock.json
+++ b/package-lock.json
@@ -8,13 +8,16 @@
      "name": "comfyui-frontend",
      "version": "1.0.0",
      "dependencies": {
+        "dotenv": "^16.4.5",
        "zod": "^3.23.8",
        "zod-validation-error": "^3.3.0"
      },
      "devDependencies": {
        "@babel/core": "^7.24.7",
        "@babel/preset-env": "^7.22.20",
+        "@playwright/test": "^1.44.1",
        "@types/jest": "^29.5.12",
+        "@types/node": "^20.14.8",
        "babel-plugin-transform-import-meta": "^2.2.1",
        "babel-plugin-transform-rename-import": "^2.3.0",
        "identity-obj-proxy": "^3.0.0",
@@ -2894,6 +2897,21 @@
        "node": ">= 8"
      }
    },
+    "node_modules/@playwright/test": {
+      "version": "1.44.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.44.1.tgz",
+      "integrity": "sha512-1hZ4TNvD5z9VuhNJ/walIjvMVvYkZKf71axoF/uiAqpntQJXpG64dlXhoDXE3OczPuTuvjf/M5KWFg5VAVUS3Q==",
+      "dev": true,
+      "dependencies": {
+        "playwright": "1.44.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
    "node_modules/@rollup/rollup-android-arm-eabi": {
      "version": "4.18.0",
      "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.18.0.tgz",
@@ -3261,9 +3279,9 @@
      }
    },
    "node_modules/@types/node": {
-      "version": "20.14.2",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.2.tgz",
-      "integrity": "sha512-xyu6WAMVwv6AKFLB+e/7ySZVr/0zLCzOa7rSpq6jNwpqOrUbcACDWC+53d4n2QHOnDou0fbIsg8wZu/sxrnI4Q==",
+      "version": "20.14.8",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.8.tgz",
+      "integrity": "sha512-DO+2/jZinXfROG7j7WKFn/3C6nFwxy2lLpgLjEXJz+0XKphZlTLJ14mo8Vfg8X5BWN6XjyESXq+LcYdT7tR3bA==",
      "dev": true,
      "dependencies": {
        "undici-types": "~5.26.4"
@@ -4185,6 +4203,17 @@
        "node": ">=12"
      }
    },
+    "node_modules/dotenv": {
+      "version": "16.4.5",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz",
+      "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://dotenvx.com"
+      }
+    },
    "node_modules/electron-to-chromium": {
      "version": "1.4.803",
      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.803.tgz",
@@ -7080,6 +7109,50 @@
        "node": ">=8"
      }
    },
+    "node_modules/playwright": {
+      "version": "1.44.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.44.1.tgz",
+      "integrity": "sha512-qr/0UJ5CFAtloI3avF95Y0L1xQo6r3LQArLIg/z/PoGJ6xa+EwzrwO5lpNr/09STxdHuUoP2mvuELJS+hLdtgg==",
+      "dev": true,
+      "dependencies": {
+        "playwright-core": "1.44.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.44.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.44.1.tgz",
+      "integrity": "sha512-wh0JWtYTrhv1+OSsLPgFzGzt67Y7BE/ZS3jEqgGBlp2ppp1ZDj8c+9IARNW4dwf1poq5MgHreEM2KV/GuR4cFA==",
+      "dev": true,
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
+    "node_modules/playwright/node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
    "node_modules/postcss": {
      "version": "8.4.38",
      "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.38.tgz",
--- a/package.json
+++ b/package.json
@@ -9,12 +9,15 @@
    "typecheck": "tsc --noEmit",
    "test": "npm run build && jest",
    "test:generate": "npx tsx tests-ui/setup",
+    "test:browser": "npx playwright test",
    "preview": "vite preview"
  },
  "devDependencies": {
    "@babel/core": "^7.24.7",
    "@babel/preset-env": "^7.22.20",
+    "@playwright/test": "^1.44.1",
    "@types/jest": "^29.5.12",
+    "@types/node": "^20.14.8",
    "babel-plugin-transform-import-meta": "^2.2.1",
    "babel-plugin-transform-rename-import": "^2.3.0",
    "identity-obj-proxy": "^3.0.0",
@@ -28,6 +31,7 @@
    "vite-plugin-static-copy": "^1.0.5"
  },
  "dependencies": {
+    "dotenv": "^16.4.5",
    "zod": "^3.23.8",
    "zod-validation-error": "^3.3.0"
  }
--- a/playwright.config.ts
+++ b/playwright.config.ts
@@ -0,0 +1,78 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './browser_tests',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+
+    // {
+    //   name: 'firefox',
+    //   use: { ...devices['Desktop Firefox'] },
+    // },
+
+    // {
+    //   name: 'webkit',
+    //   use: { ...devices['Desktop Safari'] },
+    // },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://127.0.0.1:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+});