feat: record RUM workflow execution vitals

src/platform/telemetry/TelemetryRegistry.ts

@@ -7,9 +7,8 @@ import type {
   BeginCheckoutMetadata,
   DefaultViewSetMetadata,
   EnterLinearMetadata,
-  ShareFlowMetadata,
-  ShareLinkOpenedMetadata,
   ExecutionErrorMetadata,
+  ExecutionOutcomeMetadata,
   ExecutionSuccessMetadata,
   HelpCenterClosedMetadata,
   HelpCenterOpenedMetadata,
@@ -22,6 +21,8 @@ import type {
   PageVisibilityMetadata,
   ResubscribeClickMetadata,
   RunButtonProperties,
+  ShareFlowMetadata,
+  ShareLinkOpenedMetadata,
   SettingChangedMetadata,
   SharedWorkflowRunMetadata,
   ShellLayoutMetadata,
@@ -268,6 +269,10 @@ export class TelemetryRegistry implements TelemetryDispatcher {
     this.dispatch((provider) => provider.trackWorkflowExecution?.())
   }
 
+  trackExecutionOutcome(metadata: ExecutionOutcomeMetadata): void {
+    this.dispatch((provider) => provider.trackExecutionOutcome?.(metadata))
+  }
+
   trackExecutionError(metadata: ExecutionErrorMetadata): void {
     this.dispatch((provider) => provider.trackExecutionError?.(metadata))
   }

src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts

@@ -2,94 +2,47 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
 
 import { DatadogRumTelemetryProvider } from './DatadogRumTelemetryProvider'
 
-const addAction = vi.fn()
-const setViewContextProperty = vi.fn()
+const addDurationVital = vi.fn()
 
 function installDatadogRum(): void {
   Object.defineProperty(window, 'DD_RUM', {
     configurable: true,
-    value: { addAction, setViewContextProperty }
+    value: { addDurationVital }
   })
 }
 
-function setNavigationType(type?: NavigationTimingType): void {
-  vi.spyOn(performance, 'getEntriesByType').mockReturnValue(
-    type ? ([{ type }] as PerformanceNavigationTiming[]) : []
-  )
-}
-
 afterEach(() => {
-  addAction.mockReset()
-  setViewContextProperty.mockReset()
   vi.restoreAllMocks()
+  vi.clearAllMocks()
   Reflect.deleteProperty(window, 'DD_RUM')
 })
 
 describe('DatadogRumTelemetryProvider', () => {
-  it.for([
-    { expected: 'navigate', type: 'navigate' },
-    { expected: 'reload', type: 'reload' },
-    { expected: 'back_forward', type: 'back_forward' },
-    { expected: 'prerender', type: 'prerender' },
-    { expected: 'unknown', type: undefined }
-  ] as const)('sets $expected navigation context', ({ expected, type }) => {
+  it('records a workflow vital with its duration and outcome', () => {
     installDatadogRum()
-    setNavigationType(type)
+    vi.spyOn(performance, 'now').mockReturnValue(142)
 
-    new DatadogRumTelemetryProvider()
+    new DatadogRumTelemetryProvider().trackExecutionOutcome({
+      startTime: 42,
+      outcome: 'success'
+    })
 
-    expect(setViewContextProperty).toHaveBeenCalledWith(
-      'navigation_type',
-      expected
-    )
-  })
-
-  it('tracks workflow execution starts', () => {
-    installDatadogRum()
-    setNavigationType('navigate')
-
-    new DatadogRumTelemetryProvider().trackWorkflowExecution()
-
-    expect(addAction).toHaveBeenCalledWith('workflow_execution_started', {
-      product: 'cloud_generation',
-      product_surface: 'workspace'
+    expect(addDurationVital).toHaveBeenCalledWith('workflow_execution', {
+      startTime: performance.timeOrigin + 42,
+      duration: 100,
+      context: {
+        outcome: 'success',
+        product: 'cloud_generation'
+      }
     })
   })
 
-  it.for([
-    {
-      outcome: 'success',
-      trackOutcome: (provider: DatadogRumTelemetryProvider) =>
-        provider.trackExecutionSuccess()
-    },
-    {
-      outcome: 'failure',
-      trackOutcome: (provider: DatadogRumTelemetryProvider) =>
-        provider.trackExecutionError()
-    }
-  ] as const)(
-    'tracks workflow $outcome outcomes',
-    ({ outcome, trackOutcome }) => {
-      installDatadogRum()
-      setNavigationType('navigate')
-      const provider = new DatadogRumTelemetryProvider()
-
-      trackOutcome(provider)
-
-      expect(addAction).toHaveBeenCalledWith('workflow_execution_completed', {
-        outcome,
-        product: 'cloud_generation',
-        product_surface: 'workspace'
-      })
-    }
-  )
-
   it('does nothing when Datadog RUM is unavailable', () => {
-    const provider = new DatadogRumTelemetryProvider()
-
-    expect(() => provider.trackWorkflowExecution()).not.toThrow()
-    expect(() => provider.trackExecutionSuccess()).not.toThrow()
-    expect(() => provider.trackExecutionError()).not.toThrow()
-    expect(addAction).not.toHaveBeenCalled()
+    expect(() =>
+      new DatadogRumTelemetryProvider().trackExecutionOutcome({
+        startTime: 42,
+        outcome: 'failure'
+      })
+    ).not.toThrow()
   })
 })

src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts

@@ -1,54 +1,35 @@
-import type { TelemetryProvider } from '../../types'
+import type { ExecutionOutcomeMetadata, TelemetryProvider } from '../../types'
+
+interface DatadogRumDurationVitalOptions {
+  startTime: number
+  duration: number
+  context?: Record<string, unknown>
+}
 
 interface DatadogRumClient {
-  addAction(name: string, context?: Record<string, unknown>): void
-  setViewContextProperty(name: string, value: unknown): void
+  addDurationVital(name: string, options: DatadogRumDurationVitalOptions): void
 }
 
 interface WindowWithDatadogRum extends Window {
   DD_RUM?: DatadogRumClient
 }
 
-const WORKFLOW_CONTEXT = {
-  product: 'cloud_generation',
-  product_surface: 'workspace'
-} as const
-
 function getDatadogRum(): DatadogRumClient | undefined {
   return (window as WindowWithDatadogRum).DD_RUM
 }
 
-function getNavigationType(): NavigationTimingType | 'unknown' {
-  const [navigation] = performance.getEntriesByType(
-    'navigation'
-  ) as PerformanceNavigationTiming[]
-  return navigation?.type ?? 'unknown'
-}
-
 export class DatadogRumTelemetryProvider implements TelemetryProvider {
-  constructor() {
-    getDatadogRum()?.setViewContextProperty(
-      'navigation_type',
-      getNavigationType()
-    )
-  }
-
-  trackWorkflowExecution(): void {
-    getDatadogRum()?.addAction('workflow_execution_started', WORKFLOW_CONTEXT)
-  }
-
-  trackExecutionSuccess(): void {
-    this.trackTerminalOutcome('success')
-  }
-
-  trackExecutionError(): void {
-    this.trackTerminalOutcome('failure')
-  }
-
-  private trackTerminalOutcome(outcome: 'success' | 'failure'): void {
-    getDatadogRum()?.addAction('workflow_execution_completed', {
-      ...WORKFLOW_CONTEXT,
-      outcome
+  trackExecutionOutcome({
+    startTime,
+    outcome
+  }: ExecutionOutcomeMetadata): void {
+    getDatadogRum()?.addDurationVital('workflow_execution', {
+      startTime: performance.timeOrigin + startTime,
+      duration: performance.now() - startTime,
+      context: {
+        outcome,
+        product: 'cloud_generation'
+      }
     })
   }
 }

src/platform/telemetry/types.ts

@@ -156,6 +156,11 @@ export interface ExecutionErrorMetadata {
   error?: string
 }
 
+export interface ExecutionOutcomeMetadata {
+  startTime: number
+  outcome: 'success' | 'failure'
+}
+
 /**
  * Execution success metadata
  */
@@ -632,6 +637,7 @@ export interface TelemetryProvider {
 
   // Workflow execution events
   trackWorkflowExecution?(): void
+  trackExecutionOutcome?(metadata: ExecutionOutcomeMetadata): void
   trackExecutionError?(metadata: ExecutionErrorMetadata): void
   trackExecutionSuccess?(metadata: ExecutionSuccessMetadata): void
   trackSharedWorkflowRun?(metadata: SharedWorkflowRunMetadata): void

src/scripts/app.ts

@@ -1662,6 +1662,7 @@ export class ComfyApp {
           // user switches tabs while the request is in flight.
           const queuedWorkflow = useWorkspaceStore().workflow
             .activeWorkflow as ComfyWorkflow
+          const startTime = performance.now()
           const p = await this.graphToPrompt(this.rootGraph)
           const queuedNodes = collectAllNodes(this.rootGraph)
           try {
@@ -1685,6 +1686,7 @@ export class ComfyApp {
                   id: res.prompt_id,
                   nodes: Object.keys(p.output),
                   promptOutput: p.output,
+                  startTime,
                   workflow: queuedWorkflow
                 })
               }

src/stores/executionStore.test.ts

@@ -21,6 +21,7 @@ const {
   mockOpenWorkflows,
   mockShowTextPreview,
   mockTrackExecutionError,
+  mockTrackExecutionOutcome,
   mockTrackExecutionSuccess,
   mockTrackSharedWorkflowRun
 } = await vi.hoisted(async () => {
@@ -33,6 +34,7 @@ const {
     mockOpenWorkflows: shallowRef<{ path: string }[]>([]),
     mockShowTextPreview: vi.fn(),
     mockTrackExecutionError: vi.fn(),
+    mockTrackExecutionOutcome: vi.fn(),
     mockTrackExecutionSuccess: vi.fn(),
     mockTrackSharedWorkflowRun: vi.fn()
   }
@@ -92,6 +94,7 @@ vi.mock('@/platform/distribution/types', async () => ({
 vi.mock('@/platform/telemetry', () => ({
   useTelemetry: () => ({
     trackExecutionError: mockTrackExecutionError,
+    trackExecutionOutcome: mockTrackExecutionOutcome,
     trackExecutionSuccess: mockTrackExecutionSuccess,
     trackSharedWorkflowRun: mockTrackSharedWorkflowRun
   })
@@ -614,6 +617,7 @@ describe('useExecutionStore - workflowStatus', () => {
       nodes: ['1'],
       id: jobId,
       promptOutput: { '1': createPromptNode('Node', 'TestNode') },
+      startTime: 42,
       workflow
     })
   }
@@ -631,6 +635,7 @@ describe('useExecutionStore - workflowStatus', () => {
     callStoreJob('job-1', workflowA)
     fireExecutionStart('job-1')
 
+    expect(mockTrackExecutionOutcome).not.toHaveBeenCalled()
     expect(store.getWorkflowStatus(workflowA)).toBe('running')
   })
 
@@ -648,6 +653,11 @@ describe('useExecutionStore - workflowStatus', () => {
     fireExecutionSuccess('job-1')
 
     callStoreJob('job-1', workflowA)
+    expect(mockTrackExecutionOutcome).toHaveBeenCalledOnce()
+    expect(mockTrackExecutionOutcome).toHaveBeenCalledWith({
+      startTime: 42,
+      outcome: 'success'
+    })
     expect(store.getWorkflowStatus(workflowA)).toBe('completed')
   })
 
@@ -656,6 +666,10 @@ describe('useExecutionStore - workflowStatus', () => {
     fireExecutionError('job-1')
 
     callStoreJob('job-1', workflowA)
+    expect(mockTrackExecutionOutcome).toHaveBeenCalledWith({
+      startTime: 42,
+      outcome: 'failure'
+    })
     expect(store.getWorkflowStatus(workflowA)).toBe('failed')
   })
 
@@ -672,6 +686,10 @@ describe('useExecutionStore - workflowStatus', () => {
     fireExecutionStart('job-1')
     fireExecutionSuccess('job-1')
 
+    expect(mockTrackExecutionOutcome).toHaveBeenCalledWith({
+      startTime: 42,
+      outcome: 'success'
+    })
     expect(store.getWorkflowStatus(workflowA)).toBe('completed')
   })
 

src/stores/executionStore.ts

@@ -51,6 +51,7 @@ interface QueuedJob {
    * value is a boolean indicating if the node has been executed.
    */
   nodes: Record<string, boolean>
+  startTime?: number
   /**
    * The workflow that is queued to be executed
    */
@@ -177,6 +178,19 @@ export const useExecutionStore = defineStore('execution', () => {
     mutateStatus((m) => m.set(workflow, status))
   }
 
+  function trackExecutionOutcome(
+    jobId: string,
+    status: WorkflowExecutionStatus
+  ) {
+    if (status === 'running') return
+    const startTime = queuedJobs.value[jobId]?.startTime
+    if (startTime === undefined) return
+    useTelemetry()?.trackExecutionOutcome({
+      startTime,
+      outcome: status === 'completed' ? 'success' : 'failure'
+    })
+  }
+
   function setWorkflowStatus(jobId: string, status: WorkflowExecutionStatus) {
     const workflow = jobIdToWorkflow.get(jobId)
     if (!workflow) {
@@ -184,6 +198,7 @@ export const useExecutionStore = defineStore('execution', () => {
       return
     }
     applyWorkflowStatus(workflow, status)
+    trackExecutionOutcome(jobId, status)
   }
 
   function clearWorkflowStatus(workflow: ComfyWorkflow) {
@@ -718,11 +733,13 @@ export const useExecutionStore = defineStore('execution', () => {
     nodes,
     id,
     promptOutput,
+    startTime,
     workflow
   }: {
     nodes: string[]
     id: JobId
     promptOutput: ComfyApiWorkflow
+    startTime?: number
     workflow: ComfyWorkflow
   }) {
     queuedJobs.value[id] ??= { nodes: {} }
@@ -735,6 +752,7 @@ export const useExecutionStore = defineStore('execution', () => {
       ...queuedJob.nodes
     }
     queuedJob.nodeLookup = buildExecutionNodeLookup(promptOutput)
+    queuedJob.startTime = startTime
     queuedJob.workflow = workflow
     if (workflow) jobIdToWorkflow.set(String(id), workflow)
     queuedJob.shareId = workflow?.shareId
@@ -761,6 +779,7 @@ export const useExecutionStore = defineStore('execution', () => {
     // Don't let a stale 'running' overwrite a terminal status already set.
     if (pending === 'running' && workflowStatus.value.has(workflow)) return
     applyWorkflowStatus(workflow, pending)
+    trackExecutionOutcome(jobId, pending)
   }
 
   // ~0.65 MB at capacity (32 char GUID key + 50 char path value)