mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-04-19 22:09:37 +00:00
bce7a168de
## Motivation Browser tests mock API responses with `route.fulfill()` using untyped inline JSON. When the OpenAPI spec changes, these mocks silently drift — mismatches aren't caught at compile time and only surface as test failures at runtime. We already have auto-generated types from OpenAPI and manual Zod schemas. This PR makes those types the source of truth for test mock data. From Mar 27 PR review session action item: "instruct agents to use schemas and types when writing browser tests." ## Type packages and their API coverage The frontend has two OpenAPI-generated type packages, each targeting a different backend API with a different code generation tool: | Package | Target API | Generator | TS types | Zod schemas | |---------|-----------|-----------|----------|-------------| | `@comfyorg/registry-types` | Registry API (node packages, releases, subscriptions, customers) | `openapi-typescript` | Yes | **No** | | `@comfyorg/ingest-types` | Ingest API (hub workflows, asset uploads, workspaces) | `@hey-api/openapi-ts` | Yes | Yes | Additionally, Python backend endpoints (`/api/queue`, `/api/features`, `/api/settings`, etc.) are typed via manual Zod schemas in `src/schemas/apiSchema.ts`. This PR applies **compile-time type checking** using these existing types. Runtime validation via Zod `.parse()` is not yet possible for all endpoints because `registry-types` does not generate Zod schemas — this requires a separate migration of `registry-types` to `@hey-api/openapi-ts` (#10674). ## Summary - Add "Typed API Mocks" guideline to `docs/guidance/playwright.md` with a sources-of-truth table mapping endpoint categories to their type packages - Add rule to `AGENTS.md` Playwright section requiring typed mock data - Refactor `releaseNotifications.spec.ts` to use `ReleaseNote` type (from `registry-types`) via `createMockRelease()` factory - Annotate template mock in `templates.spec.ts` with `WorkflowTemplates[]` type Refs #10656 ## Example workflow: writing a new typed E2E test mock When adding a new `route.fulfill()` mock, follow these steps: ### 1. Identify the type source Check which API the endpoint belongs to: | Endpoint category | Type source | Zod available | |---|---|---| | Ingest API (hub, billing, workflows) | `@comfyorg/ingest-types` | Yes — use `.parse()` | | Registry API (releases, nodes, publishers) | `@comfyorg/registry-types` | Not yet (#10674) — TS type only | | Python backend (queue, history, settings) | `src/schemas/apiSchema.ts` | Yes — use `z.infer` | | Templates | `src/platform/workflow/templates/types/template.ts` | No — TS type only | ### 2. Create a typed factory (with Zod when available) **Ingest API endpoints** — Zod schemas exist, use `.parse()` for runtime validation: ```typescript import { zBillingStatusResponse } from '@comfyorg/ingest-types/zod' import type { BillingStatusResponse } from '@comfyorg/ingest-types' function createMockBillingStatus( overrides?: Partial<BillingStatusResponse> ): BillingStatusResponse { return zBillingStatusResponse.parse({ plan: 'free', credits_remaining: 100, renewal_date: '2026-04-28T00:00:00Z', ...overrides }) } ``` **Registry API endpoints** — TS type only (Zod not yet generated): ```typescript import type { ReleaseNote } from '../../src/platform/updates/common/releaseService' function createMockRelease( overrides?: Partial<ReleaseNote> ): ReleaseNote { return { id: 1, project: 'comfyui', version: 'v0.3.44', attention: 'medium', content: '## New Features', published_at: new Date().toISOString(), ...overrides } } ``` ### 3. Use in test ```typescript test('should show upgrade banner for free plan', async ({ comfyPage }) => { await comfyPage.page.route('**/billing/status', async (route) => { await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(createMockBillingStatus({ plan: 'free' })) }) }) await comfyPage.setup() await expect(comfyPage.page.getByText('Upgrade')).toBeVisible() }) ``` The factory pattern keeps test bodies focused on **what varies** (the override) rather than the full response shape. ## Scope decisions | File | Decision | Reason | |------|----------|--------| | `releaseNotifications.spec.ts` | Typed | `ReleaseNote` type available from `registry-types` | | `templates.spec.ts` | Typed | `WorkflowTemplates` type available in `src/platform/workflow/templates/types/` | | `QueueHelper.ts` | Skipped | Dead code — instantiated but never called in any test | | `FeatureFlagHelper.ts` | Skipped | Response type is inherently `Record<string, unknown>`, no stronger type exists | | Fixture factories | Deferred | Coordinate with Ben's fixture restructuring work to avoid duplication | ## Follow-up work Sub-issues of #10656: - #10670 — Clean up dead `QueueHelper` or rewrite against `/api/jobs` endpoint - #10671 — Expand typed factory pattern to more endpoints - #10672 — Evaluate OpenAPI generation for excluded Python backend endpoints - #10674 — Migrate `registry-types` from `openapi-typescript` to `@hey-api/openapi-ts` to enable Zod schema generation ## Test plan - [x] `pnpm typecheck:browser` passes - [x] `pnpm lint` passes - [ ] Existing `releaseNotifications` and `templates` tests pass in CI
387 lines
11 KiB
TypeScript
387 lines
11 KiB
TypeScript
import { expect } from '@playwright/test'
|
|
|
|
import type { components } from '@comfyorg/registry-types'
|
|
|
|
type ReleaseNote = components['schemas']['ReleaseNote']
|
|
import { comfyPageFixture as test } from '../fixtures/ComfyPage'
|
|
import { TestIds } from '../fixtures/selectors'
|
|
|
|
function createMockRelease(overrides?: Partial<ReleaseNote>): ReleaseNote {
|
|
return {
|
|
id: 1,
|
|
project: 'comfyui',
|
|
version: 'v0.3.44',
|
|
attention: 'medium',
|
|
content: '## New Features\n\n- Added awesome feature',
|
|
published_at: new Date().toISOString(),
|
|
...overrides
|
|
}
|
|
}
|
|
|
|
test.describe('Release Notifications', () => {
|
|
test.beforeEach(async ({ comfyPage }) => {
|
|
await comfyPage.settings.setSetting('Comfy.UseNewMenu', 'Top')
|
|
})
|
|
|
|
test('should show help center with release information', async ({
|
|
comfyPage
|
|
}) => {
|
|
// Mock release API with test data instead of empty array
|
|
await comfyPage.page.route('**/releases**', async (route) => {
|
|
const url = route.request().url()
|
|
if (
|
|
url.includes('api.comfy.org') ||
|
|
url.includes('stagingapi.comfy.org')
|
|
) {
|
|
await route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify([
|
|
createMockRelease({
|
|
content:
|
|
'## New Features\n\n- Added awesome feature\n- Fixed important bug'
|
|
})
|
|
])
|
|
})
|
|
} else {
|
|
await route.continue()
|
|
}
|
|
})
|
|
|
|
// Setup with release mocking disabled for this test
|
|
await comfyPage.setup({ mockReleases: false })
|
|
|
|
// Open help center
|
|
const helpCenterButton = comfyPage.page.locator('.comfy-help-center-btn')
|
|
await helpCenterButton.waitFor({ state: 'visible' })
|
|
await helpCenterButton.click()
|
|
|
|
// Verify help center menu appears
|
|
const helpMenu = comfyPage.page.locator('.help-center-menu')
|
|
await expect(helpMenu).toBeVisible()
|
|
|
|
// Verify "What's New?" section shows the release
|
|
const whatsNewSection = comfyPage.page.getByTestId(
|
|
TestIds.dialogs.whatsNewSection
|
|
)
|
|
await expect(whatsNewSection).toBeVisible()
|
|
|
|
// Should show the release version
|
|
await expect(
|
|
whatsNewSection.locator('text=Comfy v0.3.44 Release')
|
|
).toBeVisible()
|
|
|
|
// Close help center by dismissable mask
|
|
await comfyPage.page.click('.help-center-backdrop')
|
|
await expect(helpMenu).not.toBeVisible()
|
|
})
|
|
|
|
test('should not show release notifications when mocked (default behavior)', async ({
|
|
comfyPage
|
|
}) => {
|
|
// Use default setup (mockReleases: true)
|
|
await comfyPage.setup()
|
|
|
|
// Open help center
|
|
const helpCenterButton = comfyPage.page.locator('.comfy-help-center-btn')
|
|
await helpCenterButton.waitFor({ state: 'visible' })
|
|
await helpCenterButton.click()
|
|
|
|
// Verify help center menu appears
|
|
const helpMenu = comfyPage.page.locator('.help-center-menu')
|
|
await expect(helpMenu).toBeVisible()
|
|
|
|
// Verify "What's New?" section shows no releases
|
|
const whatsNewSection = comfyPage.page.getByTestId(
|
|
TestIds.dialogs.whatsNewSection
|
|
)
|
|
await expect(whatsNewSection).toBeVisible()
|
|
|
|
// Should show "No recent releases" message
|
|
await expect(
|
|
whatsNewSection.locator('text=No recent releases')
|
|
).toBeVisible()
|
|
|
|
// Should not show any popups or toasts
|
|
await expect(comfyPage.page.locator('.whats-new-popup')).not.toBeVisible()
|
|
await expect(
|
|
comfyPage.page.locator('.release-notification-toast')
|
|
).not.toBeVisible()
|
|
})
|
|
|
|
test('should handle release API errors gracefully', async ({ comfyPage }) => {
|
|
// Mock API to return an error
|
|
await comfyPage.page.route('**/releases**', async (route) => {
|
|
const url = route.request().url()
|
|
if (
|
|
url.includes('api.comfy.org') ||
|
|
url.includes('stagingapi.comfy.org')
|
|
) {
|
|
await route.fulfill({
|
|
status: 500,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ error: 'Server error' })
|
|
})
|
|
} else {
|
|
await route.continue()
|
|
}
|
|
})
|
|
|
|
// Setup with release mocking disabled
|
|
await comfyPage.setup({ mockReleases: false })
|
|
|
|
// Open help center
|
|
const helpCenterButton = comfyPage.page.locator('.comfy-help-center-btn')
|
|
await helpCenterButton.waitFor({ state: 'visible' })
|
|
await helpCenterButton.click()
|
|
|
|
// Verify help center still works despite API error
|
|
const helpMenu = comfyPage.page.locator('.help-center-menu')
|
|
await expect(helpMenu).toBeVisible()
|
|
|
|
// Should show no releases due to error
|
|
const whatsNewSection = comfyPage.page.getByTestId(
|
|
TestIds.dialogs.whatsNewSection
|
|
)
|
|
await expect(
|
|
whatsNewSection.locator('text=No recent releases')
|
|
).toBeVisible()
|
|
})
|
|
|
|
test('should hide "What\'s New" section when notifications are disabled', async ({
|
|
comfyPage
|
|
}) => {
|
|
// Disable version update notifications
|
|
await comfyPage.settings.setSetting(
|
|
'Comfy.Notification.ShowVersionUpdates',
|
|
false
|
|
)
|
|
|
|
// Mock release API with test data
|
|
await comfyPage.page.route('**/releases**', async (route) => {
|
|
const url = route.request().url()
|
|
if (
|
|
url.includes('api.comfy.org') ||
|
|
url.includes('stagingapi.comfy.org')
|
|
) {
|
|
await route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify([createMockRelease({ attention: 'high' })])
|
|
})
|
|
} else {
|
|
await route.continue()
|
|
}
|
|
})
|
|
|
|
await comfyPage.setup({ mockReleases: false })
|
|
|
|
// Open help center
|
|
const helpCenterButton = comfyPage.page.locator('.comfy-help-center-btn')
|
|
await helpCenterButton.waitFor({ state: 'visible' })
|
|
await helpCenterButton.click()
|
|
|
|
// Verify help center menu appears
|
|
const helpMenu = comfyPage.page.locator('.help-center-menu')
|
|
await expect(helpMenu).toBeVisible()
|
|
|
|
// Verify "What's New?" section is hidden
|
|
const whatsNewSection = comfyPage.page.getByTestId(
|
|
TestIds.dialogs.whatsNewSection
|
|
)
|
|
await expect(whatsNewSection).not.toBeVisible()
|
|
|
|
// Should not show any popups or toasts
|
|
await expect(comfyPage.page.locator('.whats-new-popup')).not.toBeVisible()
|
|
await expect(
|
|
comfyPage.page.locator('.release-notification-toast')
|
|
).not.toBeVisible()
|
|
})
|
|
|
|
test('should not make API calls when notifications are disabled', async ({
|
|
comfyPage
|
|
}) => {
|
|
// Disable version update notifications
|
|
await comfyPage.settings.setSetting(
|
|
'Comfy.Notification.ShowVersionUpdates',
|
|
false
|
|
)
|
|
|
|
// Track API calls
|
|
let apiCallCount = 0
|
|
await comfyPage.page.route('**/releases**', async (route) => {
|
|
const url = route.request().url()
|
|
if (
|
|
url.includes('api.comfy.org') ||
|
|
url.includes('stagingapi.comfy.org')
|
|
) {
|
|
apiCallCount++
|
|
await route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify([])
|
|
})
|
|
} else {
|
|
await route.continue()
|
|
}
|
|
})
|
|
|
|
await comfyPage.setup({ mockReleases: false })
|
|
|
|
// Verify no API calls were made
|
|
expect(apiCallCount).toBe(0)
|
|
})
|
|
|
|
test('should show "What\'s New" section when notifications are enabled', async ({
|
|
comfyPage
|
|
}) => {
|
|
// Enable version update notifications (default behavior)
|
|
await comfyPage.settings.setSetting(
|
|
'Comfy.Notification.ShowVersionUpdates',
|
|
true
|
|
)
|
|
|
|
// Mock release API with test data
|
|
await comfyPage.page.route('**/releases**', async (route) => {
|
|
const url = route.request().url()
|
|
if (
|
|
url.includes('api.comfy.org') ||
|
|
url.includes('stagingapi.comfy.org')
|
|
) {
|
|
await route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify([createMockRelease()])
|
|
})
|
|
} else {
|
|
await route.continue()
|
|
}
|
|
})
|
|
|
|
await comfyPage.setup({ mockReleases: false })
|
|
|
|
// Open help center
|
|
const helpCenterButton = comfyPage.page.locator('.comfy-help-center-btn')
|
|
await helpCenterButton.waitFor({ state: 'visible' })
|
|
await helpCenterButton.click()
|
|
|
|
// Verify help center menu appears
|
|
const helpMenu = comfyPage.page.locator('.help-center-menu')
|
|
await expect(helpMenu).toBeVisible()
|
|
|
|
// Verify "What's New?" section is visible
|
|
const whatsNewSection = comfyPage.page.getByTestId(
|
|
TestIds.dialogs.whatsNewSection
|
|
)
|
|
await expect(whatsNewSection).toBeVisible()
|
|
|
|
// Should show the release
|
|
await expect(
|
|
whatsNewSection.locator('text=Comfy v0.3.44 Release')
|
|
).toBeVisible()
|
|
})
|
|
|
|
test('should toggle "What\'s New" section when setting changes', async ({
|
|
comfyPage
|
|
}) => {
|
|
// Mock release API with test data
|
|
await comfyPage.page.route('**/releases**', async (route) => {
|
|
const url = route.request().url()
|
|
if (
|
|
url.includes('api.comfy.org') ||
|
|
url.includes('stagingapi.comfy.org')
|
|
) {
|
|
await route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify([
|
|
createMockRelease({
|
|
attention: 'low',
|
|
content: '## Bug Fixes\n\n- Fixed minor issue'
|
|
})
|
|
])
|
|
})
|
|
} else {
|
|
await route.continue()
|
|
}
|
|
})
|
|
|
|
// Start with notifications enabled
|
|
await comfyPage.settings.setSetting(
|
|
'Comfy.Notification.ShowVersionUpdates',
|
|
true
|
|
)
|
|
await comfyPage.setup({ mockReleases: false })
|
|
|
|
// Open help center
|
|
const helpCenterButton = comfyPage.page.locator('.comfy-help-center-btn')
|
|
await helpCenterButton.waitFor({ state: 'visible' })
|
|
await helpCenterButton.click()
|
|
|
|
// Verify "What's New?" section is visible
|
|
const whatsNewSection = comfyPage.page.getByTestId(
|
|
TestIds.dialogs.whatsNewSection
|
|
)
|
|
await expect(whatsNewSection).toBeVisible()
|
|
|
|
// Close help center
|
|
await comfyPage.page.click('.help-center-backdrop')
|
|
|
|
// Disable notifications
|
|
await comfyPage.settings.setSetting(
|
|
'Comfy.Notification.ShowVersionUpdates',
|
|
false
|
|
)
|
|
|
|
// Reopen help center
|
|
await helpCenterButton.click()
|
|
|
|
// Verify "What's New?" section is now hidden
|
|
await expect(whatsNewSection).not.toBeVisible()
|
|
})
|
|
|
|
test('should handle edge case with empty releases and disabled notifications', async ({
|
|
comfyPage
|
|
}) => {
|
|
// Disable notifications
|
|
await comfyPage.settings.setSetting(
|
|
'Comfy.Notification.ShowVersionUpdates',
|
|
false
|
|
)
|
|
|
|
// Mock empty releases
|
|
await comfyPage.page.route('**/releases**', async (route) => {
|
|
const url = route.request().url()
|
|
if (
|
|
url.includes('api.comfy.org') ||
|
|
url.includes('stagingapi.comfy.org')
|
|
) {
|
|
await route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify([])
|
|
})
|
|
} else {
|
|
await route.continue()
|
|
}
|
|
})
|
|
|
|
await comfyPage.setup({ mockReleases: false })
|
|
|
|
// Open help center
|
|
const helpCenterButton = comfyPage.page.locator('.comfy-help-center-btn')
|
|
await helpCenterButton.waitFor({ state: 'visible' })
|
|
await helpCenterButton.click()
|
|
|
|
// Verify help center still works
|
|
const helpMenu = comfyPage.page.locator('.help-center-menu')
|
|
await expect(helpMenu).toBeVisible()
|
|
|
|
// Section should be hidden regardless of empty releases
|
|
const whatsNewSection = comfyPage.page.getByTestId(
|
|
TestIds.dialogs.whatsNewSection
|
|
)
|
|
await expect(whatsNewSection).not.toBeVisible()
|
|
})
|
|
})
|