Compare commits

..

11 Commits

Author SHA1 Message Date
Matt Miller
c6a2645191 Merge branch 'main' into feat/websocket-reconnect-telemetry 2026-03-26 09:58:37 -07:00
Matt Miller
79a029d233 refactor: move WebSocket reconnect tracking from PostHog/Mixpanel to Sentry
Reconnect events are operational/debug signals, not product analytics.
Sentry is the right home for this — breadcrumbs for all reconnects,
captureMessage (warning) when jobs were in flight.

Also addresses review feedback:
- Extract to useWebSocketReconnectTracking composable
- Use performance.now() instead of Date.now()
- Add tests for the composable (6 cases)
- Remove PostHog/Mixpanel telemetry plumbing

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-24 18:38:49 -07:00
Matt Miller
1dbb26bb9a Merge branch 'main' into feat/websocket-reconnect-telemetry 2026-03-23 10:54:22 -07:00
Matt Miller
b5ba737f43 Merge branch 'main' into feat/websocket-reconnect-telemetry 2026-03-23 09:53:11 -07:00
Matt Miller
4e8729bc61 Merge branch 'main' into feat/websocket-reconnect-telemetry 2026-03-21 11:04:34 -07:00
Matt Miller
251558bd2c Merge branch 'main' into feat/websocket-reconnect-telemetry 2026-03-20 16:54:00 -07:00
Matt Miller
515f69c02f Merge branch 'main' into feat/websocket-reconnect-telemetry 2026-03-20 16:36:14 -07:00
Matt Miller
b344883871 fix: reset disconnect state on GraphView unmount
Prevents stale disconnectedAt timestamps from producing inflated
disconnect_duration_ms if the component remounts mid-disconnect.
2026-03-20 10:12:08 -07:00
Matt Miller
aec5449f62 docs: clarify activeJobsCount includes pending tasks intentionally 2026-03-20 10:12:08 -07:00
Matt Miller
53d09ab869 fix: address PR review feedback on websocket reconnect telemetry
1. Fire reconnect event unconditionally so had_active_jobs varies
   (was always true since it was inside an activeJobCount > 0 guard)
2. Capture activeJobsCount at disconnect time, not reconnect time,
   since the queue store hasn't refreshed yet on reconnect
3. Add PostHog provider tests for trackWebSocketReconnected
2026-03-20 10:12:08 -07:00
Matt Miller
92abd71213 feat: add WebSocket reconnect telemetry for incident-39 observability
Track WebSocket disconnect episodes to quantify how often users lose
real-time updates while jobs are running. Fires a single
app:websocket_reconnected event per disconnect episode, gated on
active jobs to control volume.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-20 10:12:08 -07:00
12 changed files with 228 additions and 293 deletions

View File

@@ -18,8 +18,6 @@ import { ComfyNodeSearchBoxV2 } from './components/ComfyNodeSearchBoxV2'
import { ContextMenu } from './components/ContextMenu'
import { SettingDialog } from './components/SettingDialog'
import { BottomPanel } from './components/BottomPanel'
import { ConfirmDialog } from './components/ConfirmDialog'
import { QueuePanel } from './components/QueuePanel'
import {
NodeLibrarySidebarTab,
WorkflowsSidebarTab
@@ -40,6 +38,7 @@ import { SubgraphHelper } from './helpers/SubgraphHelper'
import { ToastHelper } from './helpers/ToastHelper'
import { WorkflowHelper } from './helpers/WorkflowHelper'
import type { NodeReference } from './utils/litegraphUtils'
import type { WorkspaceStore } from '../types/globals'
dotenvConfig()
@@ -112,6 +111,48 @@ class ComfyMenu {
}
}
type KeysOfType<T, Match> = {
[K in keyof T]: T[K] extends Match ? K : never
}[keyof T]
class ConfirmDialog {
private readonly root: Locator
public readonly delete: Locator
public readonly overwrite: Locator
public readonly reject: Locator
public readonly confirm: Locator
constructor(public readonly page: Page) {
this.root = page.getByRole('dialog')
this.delete = this.root.getByRole('button', { name: 'Delete' })
this.overwrite = this.root.getByRole('button', { name: 'Overwrite' })
this.reject = this.root.getByRole('button', { name: 'Cancel' })
this.confirm = this.root.getByRole('button', { name: 'Confirm' })
}
async click(locator: KeysOfType<ConfirmDialog, Locator>) {
const loc = this[locator]
await loc.waitFor({ state: 'visible' })
await loc.click()
// Wait for the dialog mask to disappear after confirming
const mask = this.page.locator('.p-dialog-mask')
const count = await mask.count()
if (count > 0) {
await mask.first().waitFor({ state: 'hidden', timeout: 3000 })
}
// Wait for workflow service to finish if it's busy
await this.page.waitForFunction(
() =>
(window.app?.extensionManager as WorkspaceStore | undefined)?.workflow
?.isBusy === false,
undefined,
{ timeout: 3000 }
)
}
}
export class ComfyPage {
public readonly url: string
// All canvas position operations are based on default view of canvas.
@@ -150,7 +191,6 @@ export class ComfyPage {
public readonly featureFlags: FeatureFlagHelper
public readonly command: CommandHelper
public readonly bottomPanel: BottomPanel
public readonly queuePanel: QueuePanel
public readonly perf: PerformanceHelper
public readonly queue: QueueHelper
@@ -197,7 +237,6 @@ export class ComfyPage {
this.featureFlags = new FeatureFlagHelper(page)
this.command = new CommandHelper(page)
this.bottomPanel = new BottomPanel(page)
this.queuePanel = new QueuePanel(page)
this.perf = new PerformanceHelper(page)
this.queue = new QueueHelper(page)
}
@@ -471,4 +510,4 @@ export const comfyExpect = expect.extend({
message: () => `Expected element to ${isFocused ? 'not ' : ''}be focused.`
}
}
})
})

View File

@@ -1,64 +0,0 @@
import type { Locator, Page } from '@playwright/test'
import type { WorkspaceStore } from '../../types/globals'
type KeysOfType<T, Match> = {
[K in keyof T]: T[K] extends Match ? K : never
}[keyof T]
/**
* Page object for the generic confirm dialog shown via `dialogService.confirm()`.
*
* Accessible on `comfyPage.confirmDialog`.
*/
export class ConfirmDialog {
readonly root: Locator
readonly delete: Locator
readonly overwrite: Locator
/** Cancel / reject button */
readonly reject: Locator
/** Primary confirm button */
readonly confirm: Locator
constructor(public readonly page: Page) {
this.root = page.getByRole('dialog')
this.delete = this.root.getByRole('button', { name: 'Delete' })
this.overwrite = this.root.getByRole('button', { name: 'Overwrite' })
this.reject = this.root.getByRole('button', { name: 'Cancel' })
this.confirm = this.root.getByRole('button', { name: 'Confirm' })
}
async isVisible(): Promise<boolean> {
return this.root.isVisible()
}
async waitForVisible(): Promise<void> {
await this.root.waitFor({ state: 'visible' })
}
async waitForHidden(): Promise<void> {
await this.root.waitFor({ state: 'hidden' })
}
async click(locator: KeysOfType<ConfirmDialog, Locator>) {
const loc = this[locator]
await loc.waitFor({ state: 'visible' })
await loc.click()
// Wait for the dialog mask to disappear after confirming
const mask = this.page.locator('.p-dialog-mask')
const count = await mask.count()
if (count > 0) {
await mask.first().waitFor({ state: 'hidden', timeout: 3000 })
}
// Wait for workflow service to finish if it's busy
await this.page.waitForFunction(
() =>
(window.app?.extensionManager as WorkspaceStore | undefined)?.workflow
?.isBusy === false,
undefined,
{ timeout: 3000 }
)
}
}

View File

@@ -1,56 +0,0 @@
import type { Locator, Page } from '@playwright/test'
import { comfyExpect as expect } from '../ComfyPage'
import { TestIds } from '../selectors'
/**
* Page object for the "Clear queue history?" confirmation dialog that opens
* from the queue panel's history actions menu.
*/
export class QueueClearHistoryDialog {
readonly root: Locator
readonly cancelButton: Locator
readonly clearButton: Locator
readonly closeButton: Locator
constructor(public readonly page: Page) {
this.root = page.getByRole('dialog')
this.cancelButton = this.root.getByRole('button', { name: 'Cancel' })
this.clearButton = this.root.getByRole('button', { name: 'Clear' })
this.closeButton = this.root.getByLabel('Close')
}
async isVisible(): Promise<boolean> {
return this.root.isVisible()
}
async waitForVisible(): Promise<void> {
await this.root.waitFor({ state: 'visible' })
}
async waitForHidden(): Promise<void> {
await this.root.waitFor({ state: 'hidden' })
}
}
export class QueuePanel {
readonly overlayToggle: Locator
readonly moreOptionsButton: Locator
readonly clearHistoryDialog: QueueClearHistoryDialog
constructor(readonly page: Page) {
this.overlayToggle = page.getByTestId(TestIds.queue.overlayToggle)
this.moreOptionsButton = page.getByLabel(/More options/i).first()
this.clearHistoryDialog = new QueueClearHistoryDialog(page)
}
async openClearHistoryDialog() {
await this.moreOptionsButton.click()
const clearHistoryAction = this.page.getByTestId(
TestIds.queue.clearHistoryAction
)
await expect(clearHistoryAction).toBeVisible()
await clearHistoryAction.click()
}
}

View File

@@ -84,10 +84,6 @@ export const TestIds = {
user: {
currentUserIndicator: 'current-user-indicator'
},
queue: {
overlayToggle: 'queue-overlay-toggle',
clearHistoryAction: 'clear-history-action'
},
errors: {
imageLoadError: 'error-loading-image',
videoLoadError: 'error-loading-video'
@@ -116,5 +112,4 @@ export type TestIdValue =
(id: string) => string
>
| (typeof TestIds.user)[keyof typeof TestIds.user]
| (typeof TestIds.queue)[keyof typeof TestIds.queue]
| (typeof TestIds.errors)[keyof typeof TestIds.errors]

View File

@@ -18,13 +18,15 @@ test.describe('Confirm dialog text wrapping', { tag: ['@mobile'] }, () => {
.catch(() => {})
}, longFilename)
const dialog = comfyPage.confirmDialog
await dialog.waitForVisible()
const dialog = comfyPage.page.getByRole('dialog')
await expect(dialog).toBeVisible()
await expect(dialog.confirm).toBeVisible()
await expect(dialog.confirm).toBeInViewport()
const confirmButton = dialog.getByRole('button', { name: 'Confirm' })
await expect(confirmButton).toBeVisible()
await expect(confirmButton).toBeInViewport()
await expect(dialog.reject).toBeVisible()
await expect(dialog.reject).toBeInViewport()
const cancelButton = dialog.getByRole('button', { name: 'Cancel' })
await expect(cancelButton).toBeVisible()
await expect(cancelButton).toBeInViewport()
})
})
})

View File

@@ -1,137 +0,0 @@
import {
comfyPageFixture as test,
comfyExpect as expect
} from '../../fixtures/ComfyPage'
test.describe('QueueClearHistoryDialog', { tag: '@ui' }, () => {
test.beforeEach(async ({ comfyPage }) => {
await comfyPage.setup()
// Expand the queue overlay so the JobHistoryActionsMenu is visible
await comfyPage.queuePanel.overlayToggle.click()
})
test('Dialog opens from queue panel history actions menu', async ({
comfyPage
}) => {
await comfyPage.queuePanel.openClearHistoryDialog()
await expect(comfyPage.queuePanel.clearHistoryDialog.root).toBeVisible()
})
test('Dialog shows confirmation message with title, description, and assets note', async ({
comfyPage
}) => {
await comfyPage.queuePanel.openClearHistoryDialog()
const dialog = comfyPage.queuePanel.clearHistoryDialog
await expect(dialog.root).toBeVisible()
// Verify title
await expect(
dialog.root.getByText('Clear your job queue history?')
).toBeVisible()
// Verify description
await expect(
dialog.root.getByText(
'All the finished or failed jobs below will be removed from this Job queue panel.'
)
).toBeVisible()
// Verify assets note (locale uses Unicode RIGHT SINGLE QUOTATION MARK \u2019)
await expect(
dialog.root.getByText(
'Assets generated by these jobs won\u2019t be deleted and can always be viewed from the assets panel.'
)
).toBeVisible()
})
test('Cancel button closes dialog without clearing history', async ({
comfyPage
}) => {
await comfyPage.queuePanel.openClearHistoryDialog()
const dialog = comfyPage.queuePanel.clearHistoryDialog
await expect(dialog.root).toBeVisible()
// Intercept the clear API call — it should NOT be called
let clearCalled = false
await comfyPage.page.route('**/api/history', (route) => {
if (route.request().method() === 'POST') {
clearCalled = true
}
return route.continue()
})
await dialog.cancelButton.click()
await expect(dialog.root).not.toBeVisible()
expect(clearCalled).toBe(false)
await comfyPage.page.unroute('**/api/history')
})
test('Close (X) button closes dialog without clearing history', async ({
comfyPage
}) => {
await comfyPage.queuePanel.openClearHistoryDialog()
const dialog = comfyPage.queuePanel.clearHistoryDialog
await expect(dialog.root).toBeVisible()
// Intercept the clear API call — it should NOT be called
let clearCalled = false
await comfyPage.page.route('**/api/history', (route) => {
if (route.request().method() === 'POST') {
clearCalled = true
}
return route.continue()
})
await dialog.closeButton.click()
await expect(dialog.root).not.toBeVisible()
expect(clearCalled).toBe(false)
await comfyPage.page.unroute('**/api/history')
})
test('Confirm clears queue history and closes dialog', async ({
comfyPage
}) => {
await comfyPage.queuePanel.openClearHistoryDialog()
const dialog = comfyPage.queuePanel.clearHistoryDialog
await expect(dialog.root).toBeVisible()
// Intercept the clear API call to verify it is made
const clearPromise = comfyPage.page.waitForRequest(
(req) => req.url().includes('/api/history') && req.method() === 'POST'
)
await dialog.clearButton.click()
// Verify the API call was made
const request = await clearPromise
expect(request.postDataJSON()).toEqual({ clear: true })
await expect(dialog.root).not.toBeVisible()
})
test('Dialog state resets after close and reopen', async ({ comfyPage }) => {
// Open and cancel
await comfyPage.queuePanel.openClearHistoryDialog()
const dialog = comfyPage.queuePanel.clearHistoryDialog
await expect(dialog.root).toBeVisible()
await dialog.cancelButton.click()
await expect(dialog.root).not.toBeVisible()
// Reopen — dialog should be fresh (Clear button enabled, not stuck)
await comfyPage.queuePanel.openClearHistoryDialog()
await expect(dialog.root).toBeVisible()
await expect(dialog.clearButton).toBeVisible()
await expect(dialog.clearButton).toBeEnabled()
})
})

View File

@@ -10,7 +10,6 @@ import type { ComfyPage } from '../fixtures/ComfyPage'
import { DefaultGraphPositions } from '../fixtures/constants/defaultGraphPositions'
import { TestIds } from '../fixtures/selectors'
import type { NodeReference } from '../fixtures/utils/litegraphUtils'
import type { WorkspaceStore } from '../types/globals'
test.beforeEach(async ({ comfyPage }) => {
await comfyPage.settings.setSetting('Comfy.UseNewMenu', 'Disabled')
@@ -721,19 +720,6 @@ test.describe('Load workflow', { tag: '@screenshot' }, () => {
await expect(comfyPage.canvas).toHaveScreenshot('string_input.png')
})
test('Creates initial workflow tab when persistence is disabled', async ({
comfyPage
}) => {
await comfyPage.settings.setSetting('Comfy.Workflow.Persist', false)
await comfyPage.setup()
const openCount = await comfyPage.page.evaluate(() => {
return (window.app!.extensionManager as WorkspaceStore).workflow
.openWorkflows.length
})
expect(openCount).toBeGreaterThanOrEqual(1)
})
test('Restore workflow on reload (switch workflow)', async ({
comfyPage
}) => {

View File

@@ -68,7 +68,7 @@ test.describe(
})
})
test.fixme('Load workflow from URL dropped onto Vue node', async ({
test('Load workflow from URL dropped onto Vue node', async ({
comfyPage
}) => {
const fakeUrl = 'https://example.com/workflow.png'

View File

@@ -0,0 +1,101 @@
import * as Sentry from '@sentry/vue'
import { createPinia, setActivePinia } from 'pinia'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { useWebSocketReconnectTracking } from './useWebSocketReconnectTracking'
vi.mock('@sentry/vue', () => ({
addBreadcrumb: vi.fn(),
captureMessage: vi.fn()
}))
vi.mock('@/stores/queueStore', () => ({
useQueueStore: () => ({
activeJobsCount: mockActiveJobsCount
})
}))
let mockActiveJobsCount = 0
describe('useWebSocketReconnectTracking', () => {
beforeEach(() => {
vi.clearAllMocks()
mockActiveJobsCount = 0
setActivePinia(createPinia())
})
it('does nothing on reconnect without prior disconnect', () => {
const { onReconnect } = useWebSocketReconnectTracking()
onReconnect()
expect(Sentry.addBreadcrumb).not.toHaveBeenCalled()
expect(Sentry.captureMessage).not.toHaveBeenCalled()
})
it('adds breadcrumb on reconnect after disconnect', () => {
const { onDisconnect, onReconnect } = useWebSocketReconnectTracking()
onDisconnect()
onReconnect()
expect(Sentry.addBreadcrumb).toHaveBeenCalledWith(
expect.objectContaining({
category: 'websocket',
message: 'WebSocket reconnected',
level: 'info'
})
)
})
it('captures warning when jobs were active at disconnect', () => {
mockActiveJobsCount = 3
const { onDisconnect, onReconnect } = useWebSocketReconnectTracking()
onDisconnect()
onReconnect()
expect(Sentry.captureMessage).toHaveBeenCalledWith(
'WebSocket reconnected with active jobs',
expect.objectContaining({
level: 'warning',
tags: { incident: 'incident-39' },
extra: expect.objectContaining({ active_job_count: 3 })
})
)
})
it('does not capture warning when no jobs were active', () => {
mockActiveJobsCount = 0
const { onDisconnect, onReconnect } = useWebSocketReconnectTracking()
onDisconnect()
onReconnect()
expect(Sentry.addBreadcrumb).toHaveBeenCalled()
expect(Sentry.captureMessage).not.toHaveBeenCalled()
})
it('ignores duplicate disconnect calls', () => {
mockActiveJobsCount = 2
const { onDisconnect, onReconnect } = useWebSocketReconnectTracking()
onDisconnect()
// Job count changes between disconnect calls
mockActiveJobsCount = 5
onDisconnect()
onReconnect()
// Should use the count from the first disconnect (2), not the second (5)
expect(Sentry.addBreadcrumb).toHaveBeenCalledWith(
expect.objectContaining({
data: expect.objectContaining({ active_job_count: 2 })
})
)
})
it('resets state and prevents stale reconnect', () => {
const { onDisconnect, onReconnect, reset } = useWebSocketReconnectTracking()
onDisconnect()
reset()
onReconnect()
expect(Sentry.addBreadcrumb).not.toHaveBeenCalled()
})
})

View File

@@ -0,0 +1,66 @@
import * as Sentry from '@sentry/vue'
import { useQueueStore } from '@/stores/queueStore'
/**
* Tracks WebSocket disconnect/reconnect events via Sentry for incident-39
* observability. Captures disconnect duration and whether jobs were in flight
* when the connection dropped.
*/
export function useWebSocketReconnectTracking() {
const queueStore = useQueueStore()
let disconnectedAt: number | null = null
let activeJobCountAtDisconnect = 0
function onDisconnect() {
if (disconnectedAt !== null) return
disconnectedAt = performance.now()
// Includes both pending and running tasks. Pending tasks matter because
// their pending->running transition is delivered via WebSocket -- if the
// connection drops while tasks are queued, the user never sees them start.
activeJobCountAtDisconnect = queueStore.activeJobsCount
}
function onReconnect() {
if (disconnectedAt === null) return
const durationMs = Math.round(performance.now() - disconnectedAt)
const hadActiveJobs = activeJobCountAtDisconnect > 0
Sentry.addBreadcrumb({
category: 'websocket',
message: 'WebSocket reconnected',
level: 'info',
data: {
disconnect_duration_ms: durationMs,
had_active_jobs: hadActiveJobs,
active_job_count: activeJobCountAtDisconnect
}
})
if (hadActiveJobs) {
Sentry.captureMessage('WebSocket reconnected with active jobs', {
level: 'warning',
tags: {
incident: 'incident-39'
},
extra: {
disconnect_duration_ms: durationMs,
active_job_count: activeJobCountAtDisconnect
}
})
}
disconnectedAt = null
activeJobCountAtDisconnect = 0
}
function reset() {
disconnectedAt = null
activeJobCountAtDisconnect = 0
}
return { onDisconnect, onReconnect, reset }
}

View File

@@ -171,10 +171,7 @@ export function useWorkflowPersistenceV2() {
}
const initializeWorkflow = async () => {
if (!workflowPersistenceEnabled.value) {
await loadDefaultWorkflow()
return
}
if (!workflowPersistenceEnabled.value) return
try {
const restored = await loadPreviousWorkflowFromStorage()

View File

@@ -66,6 +66,7 @@ import ModelImportProgressDialog from '@/platform/assets/components/ModelImportP
import { isCloud, isDesktop } from '@/platform/distribution/types'
import { useSettingStore } from '@/platform/settings/settingStore'
import { useTelemetry } from '@/platform/telemetry'
import { useWebSocketReconnectTracking } from '@/composables/useWebSocketReconnectTracking'
import { useFrontendVersionMismatchWarning } from '@/platform/updates/common/useFrontendVersionMismatchWarning'
import { useVersionCompatibilityStore } from '@/platform/updates/common/versionCompatibilityStore'
import { useCanvasStore } from '@/renderer/core/canvas/canvasStore'
@@ -253,7 +254,10 @@ const reconnectingMessage: ToastMessageOptions = {
summary: t('g.reconnecting')
}
const wsReconnectTracking = useWebSocketReconnectTracking()
const onReconnecting = () => {
wsReconnectTracking.onDisconnect()
if (!settingStore.get('Comfy.Toast.DisableReconnectingToast')) {
toast.remove(reconnectingMessage)
toast.add(reconnectingMessage)
@@ -261,6 +265,7 @@ const onReconnecting = () => {
}
const onReconnected = () => {
wsReconnectTracking.onReconnect()
if (!settingStore.get('Comfy.Toast.DisableReconnectingToast')) {
toast.remove(reconnectingMessage)
toast.add({
@@ -288,6 +293,7 @@ onMounted(() => {
})
onBeforeUnmount(() => {
wsReconnectTracking.reset()
executionStore.unbindExecutionEvents()
})