From bf8d9de1c1fcb891ef394c8267050f473626aa7a Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Thu, 11 Dec 2025 14:02:23 -0800 Subject: [PATCH] Fix: Flaky Playwright Tests: retry some assertions (#7389) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Retries the widget value change check for up to 2 whole seconds. ┆Issue is synchronized with this [Notion page](https://www.notion.so/PR-7389-Fix-remoteWidgets-Playwright-test-add-retry-for-assertion-2c66d73d3650814e98b6fdfc83f6d3d6) by [Unito](https://www.unito.io) --- browser_tests/helpers/templates.ts | 12 ++++++++++ .../tests/backgroundImageUpload.spec.ts | 3 +-- browser_tests/tests/execution.spec.ts | 9 +++---- browser_tests/tests/remoteWidgets.spec.ts | 16 +++++++++---- .../tests/sidebar/nodeLibrary.spec.ts | 24 +++++++++++-------- browser_tests/tests/templates.spec.ts | 9 +++---- 6 files changed, 47 insertions(+), 26 deletions(-) diff --git a/browser_tests/helpers/templates.ts b/browser_tests/helpers/templates.ts index c690b8702..829837bd1 100644 --- a/browser_tests/helpers/templates.ts +++ b/browser_tests/helpers/templates.ts @@ -1,4 +1,5 @@ import type { Locator, Page } from '@playwright/test' +import { expect } from '@playwright/test' import path from 'path' import type { @@ -8,9 +9,20 @@ import type { export class ComfyTemplates { readonly content: Locator + readonly allTemplateCards: Locator constructor(readonly page: Page) { this.content = page.getByTestId('template-workflows-content') + this.allTemplateCards = page.locator('[data-testid^="template-workflow-"]') + } + + async waitForMinimumCardCount(count: number) { + return await expect(async () => { + const cardCount = await this.allTemplateCards.count() + expect(cardCount).toBeGreaterThanOrEqual(count) + }).toPass({ + timeout: 1_000 + }) } async loadTemplate(id: string) { diff --git a/browser_tests/tests/backgroundImageUpload.spec.ts b/browser_tests/tests/backgroundImageUpload.spec.ts index 44ae2d6ae..b620f5441 100644 --- a/browser_tests/tests/backgroundImageUpload.spec.ts +++ b/browser_tests/tests/backgroundImageUpload.spec.ts @@ -77,8 +77,7 @@ test.describe('Background Image Upload', () => { // Verify the URL input now has an API URL const urlInput = backgroundImageSetting.locator('input[type="text"]') - const inputValue = await urlInput.inputValue() - expect(inputValue).toMatch(/^\/api\/view\?.*subfolder=backgrounds/) + await expect(urlInput).toHaveValue(/^\/api\/view\?.*subfolder=backgrounds/) // Verify clear button is now enabled const clearButton = backgroundImageSetting.locator('button:has(.pi-trash)') diff --git a/browser_tests/tests/execution.spec.ts b/browser_tests/tests/execution.spec.ts index ed77b7949..ba22b38e9 100644 --- a/browser_tests/tests/execution.spec.ts +++ b/browser_tests/tests/execution.spec.ts @@ -36,9 +36,10 @@ test.describe('Execute to selected output nodes', () => { await output1.click('title') await comfyPage.executeCommand('Comfy.QueueSelectedOutputNodes') - - expect(await (await input.getWidget(0)).getValue()).toBe('foo') - expect(await (await output1.getWidget(0)).getValue()).toBe('foo') - expect(await (await output2.getWidget(0)).getValue()).toBe('') + await expect(async () => { + expect(await (await input.getWidget(0)).getValue()).toBe('foo') + expect(await (await output1.getWidget(0)).getValue()).toBe('foo') + expect(await (await output2.getWidget(0)).getValue()).toBe('') + }).toPass({ timeout: 2_000 }) }) }) diff --git a/browser_tests/tests/remoteWidgets.spec.ts b/browser_tests/tests/remoteWidgets.spec.ts index b717f6282..18cc77f44 100644 --- a/browser_tests/tests/remoteWidgets.spec.ts +++ b/browser_tests/tests/remoteWidgets.spec.ts @@ -212,8 +212,12 @@ test.describe('Remote COMBO Widget', () => { // Click on the canvas to trigger widget refresh await comfyPage.page.mouse.click(400, 300) - const refreshedOptions = await getWidgetOptions(comfyPage, nodeName) - expect(refreshedOptions).not.toEqual(initialOptions) + await expect(async () => { + const refreshedOptions = await getWidgetOptions(comfyPage, nodeName) + expect(refreshedOptions).not.toEqual(initialOptions) + }).toPass({ + timeout: 2_000 + }) }) test('does not refresh when TTL is not set', async ({ comfyPage }) => { @@ -321,8 +325,12 @@ test.describe('Remote COMBO Widget', () => { await clickRefreshButton(comfyPage, nodeName) // Verify the selected value of the widget is the first option in the refreshed list - const refreshedValue = await getWidgetValue(comfyPage, nodeName) - expect(refreshedValue).toEqual('new first option') + await expect(async () => { + const refreshedValue = await getWidgetValue(comfyPage, nodeName) + expect(refreshedValue).toEqual('new first option') + }).toPass({ + timeout: 2_000 + }) }) }) diff --git a/browser_tests/tests/sidebar/nodeLibrary.spec.ts b/browser_tests/tests/sidebar/nodeLibrary.spec.ts index 58f2ae448..ac7a2efa9 100644 --- a/browser_tests/tests/sidebar/nodeLibrary.spec.ts +++ b/browser_tests/tests/sidebar/nodeLibrary.spec.ts @@ -290,16 +290,20 @@ test.describe('Node library sidebar', () => { await comfyPage.page.keyboard.insertText('bar') await comfyPage.page.keyboard.press('Enter') await comfyPage.nextFrame() - expect( - await comfyPage.getSetting('Comfy.NodeLibrary.Bookmarks.V2') - ).toEqual(['bar/']) - expect( - await comfyPage.getSetting('Comfy.NodeLibrary.BookmarksCustomization') - ).toEqual({ - 'bar/': { - icon: 'pi-folder', - color: '#007bff' - } + await expect(async () => { + expect( + await comfyPage.getSetting('Comfy.NodeLibrary.Bookmarks.V2') + ).toEqual(['bar/']) + expect( + await comfyPage.getSetting('Comfy.NodeLibrary.BookmarksCustomization') + ).toEqual({ + 'bar/': { + icon: 'pi-folder', + color: '#007bff' + } + }) + }).toPass({ + timeout: 2_000 }) }) diff --git a/browser_tests/tests/templates.spec.ts b/browser_tests/tests/templates.spec.ts index 1e0d24dd6..0e2837906 100644 --- a/browser_tests/tests/templates.spec.ts +++ b/browser_tests/tests/templates.spec.ts @@ -188,22 +188,19 @@ test.describe('Templates', () => { .locator('header') .filter({ hasText: 'Templates' }) - const cardCount = await comfyPage.page - .locator('[data-testid^="template-workflow-"]') - .count() - expect(cardCount).toBeGreaterThan(0) + await comfyPage.templates.waitForMinimumCardCount(1) await expect(templateGrid).toBeVisible() await expect(nav).toBeVisible() // Nav should be visible at desktop size const mobileSize = { width: 640, height: 800 } await comfyPage.page.setViewportSize(mobileSize) - expect(cardCount).toBeGreaterThan(0) + await comfyPage.templates.waitForMinimumCardCount(1) await expect(templateGrid).toBeVisible() await expect(nav).not.toBeVisible() // Nav should collapse at mobile size const tabletSize = { width: 1024, height: 800 } await comfyPage.page.setViewportSize(tabletSize) - expect(cardCount).toBeGreaterThan(0) + await comfyPage.templates.waitForMinimumCardCount(1) await expect(templateGrid).toBeVisible() await expect(nav).toBeVisible() // Nav should be visible at tablet size })