diff --git a/src/platform/telemetry/TelemetryRegistry.ts b/src/platform/telemetry/TelemetryRegistry.ts index b4150e749a..b32983fdca 100644 --- a/src/platform/telemetry/TelemetryRegistry.ts +++ b/src/platform/telemetry/TelemetryRegistry.ts @@ -10,6 +10,7 @@ import type { ShareFlowMetadata, ShareLinkOpenedMetadata, ExecutionErrorMetadata, + ExecutionStartMetadata, ExecutionSuccessMetadata, HelpCenterClosedMetadata, HelpCenterOpenedMetadata, @@ -268,6 +269,10 @@ export class TelemetryRegistry implements TelemetryDispatcher { this.dispatch((provider) => provider.trackWorkflowExecution?.()) } + trackExecutionStarted(metadata: ExecutionStartMetadata): void { + this.dispatch((provider) => provider.trackExecutionStarted?.(metadata)) + } + trackExecutionError(metadata: ExecutionErrorMetadata): void { this.dispatch((provider) => provider.trackExecutionError?.(metadata)) } diff --git a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts index 46d8af33ed..587da30603 100644 --- a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts +++ b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts @@ -3,47 +3,24 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { DatadogRumTelemetryProvider } from './DatadogRumTelemetryProvider' const addAction = vi.fn() -const setViewContextProperty = vi.fn() +const getInternalContext = vi.fn() +const setViewName = vi.fn() function installDatadogRum(): void { Object.defineProperty(window, 'DD_RUM', { configurable: true, - value: { addAction, setViewContextProperty } + value: { addAction, getInternalContext, setViewName } }) } -function setNavigationType(type?: NavigationTimingType): void { - vi.spyOn(performance, 'getEntriesByType').mockReturnValue( - type ? ([{ type }] as PerformanceNavigationTiming[]) : [] - ) -} - afterEach(() => { addAction.mockReset() - setViewContextProperty.mockReset() - vi.restoreAllMocks() + getInternalContext.mockReset() + setViewName.mockReset() Reflect.deleteProperty(window, 'DD_RUM') }) describe('DatadogRumTelemetryProvider', () => { - it.for([ - { expected: 'navigate', type: 'navigate' }, - { expected: 'reload', type: 'reload' }, - { expected: 'back_forward', type: 'back_forward' }, - { expected: 'prerender', type: 'prerender' }, - { expected: 'unknown', type: undefined } - ] as const)('sets $expected navigation context', ({ expected, type }) => { - installDatadogRum() - setNavigationType(type) - - new DatadogRumTelemetryProvider() - - expect(setViewContextProperty).toHaveBeenCalledWith( - 'navigation_type', - expected - ) - }) - it.for([ { expected: 'workspace', path: 'https://cloud.comfy.org/' }, { expected: 'account_access', path: 'https://cloud.comfy.org/cloud/' }, @@ -68,65 +45,73 @@ describe('DatadogRumTelemetryProvider', () => { expected: 'support_recovery', path: 'https://cloud.comfy.org/cloud/auth-timeout' } - ] as const)('sets $expected product surface', ({ expected, path }) => { + ] as const)('names the current view $expected', ({ expected, path }) => { installDatadogRum() - setNavigationType('navigate') - const provider = new DatadogRumTelemetryProvider() - provider.trackPageView('ignored', { path }) + new DatadogRumTelemetryProvider().trackPageView('ignored', { path }) - expect(setViewContextProperty).toHaveBeenLastCalledWith( - 'product_surface', - expected - ) + expect(setViewName).toHaveBeenCalledWith(expected) }) - it('tracks workflow execution starts', () => { + it('tracks workflow execution starts with the originating view', () => { installDatadogRum() - setNavigationType('navigate') + getInternalContext.mockReturnValue({ view: { id: 'view-a' } }) - new DatadogRumTelemetryProvider().trackWorkflowExecution() + new DatadogRumTelemetryProvider().trackExecutionStarted({ + jobId: 'job-a', + startTime: 42 + }) + expect(getInternalContext).toHaveBeenCalledWith(42) expect(addAction).toHaveBeenCalledWith('workflow_execution_started', { product: 'cloud_generation', - product_surface: 'workspace' + origin_view_id: 'view-a' }) }) - it.for([ - { - outcome: 'success', - trackOutcome: (provider: DatadogRumTelemetryProvider) => - provider.trackExecutionSuccess() - }, - { - outcome: 'failure', - trackOutcome: (provider: DatadogRumTelemetryProvider) => - provider.trackExecutionError() - } - ] as const)( - 'tracks workflow $outcome outcomes', - ({ outcome, trackOutcome }) => { - installDatadogRum() - setNavigationType('navigate') - const provider = new DatadogRumTelemetryProvider() + it('preserves each origin view across workflow switches', () => { + installDatadogRum() + getInternalContext + .mockReturnValueOnce({ view: { id: 'view-a' } }) + .mockReturnValueOnce({ view: { id: 'view-b' } }) + const provider = new DatadogRumTelemetryProvider() - trackOutcome(provider) + provider.trackExecutionStarted({ jobId: 'job-a', startTime: 1 }) + provider.trackExecutionStarted({ jobId: 'job-b', startTime: 2 }) + provider.trackExecutionSuccess({ jobId: 'job-a' }) + provider.trackExecutionError({ jobId: 'job-b' }) - expect(addAction).toHaveBeenCalledWith('workflow_execution_completed', { - outcome, - product: 'cloud_generation', - product_surface: 'workspace' - }) - } - ) + expect(addAction).toHaveBeenNthCalledWith( + 3, + 'workflow_execution_completed', + { + outcome: 'success', + origin_view_id: 'view-a', + product: 'cloud_generation' + } + ) + expect(addAction).toHaveBeenNthCalledWith( + 4, + 'workflow_execution_completed', + { + outcome: 'failure', + origin_view_id: 'view-b', + product: 'cloud_generation' + } + ) + }) it('does nothing when Datadog RUM is unavailable', () => { const provider = new DatadogRumTelemetryProvider() - expect(() => provider.trackWorkflowExecution()).not.toThrow() - expect(() => provider.trackExecutionSuccess()).not.toThrow() - expect(() => provider.trackExecutionError()).not.toThrow() + expect(() => provider.trackPageView('ignored')).not.toThrow() + expect(() => + provider.trackExecutionStarted({ jobId: 'job-a', startTime: 1 }) + ).not.toThrow() + expect(() => + provider.trackExecutionSuccess({ jobId: 'job-a' }) + ).not.toThrow() + expect(() => provider.trackExecutionError({ jobId: 'job-a' })).not.toThrow() expect(addAction).not.toHaveBeenCalled() }) }) diff --git a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts index ce3679f708..379f4e6da8 100644 --- a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts +++ b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts @@ -1,8 +1,19 @@ -import type { PageViewMetadata, TelemetryProvider } from '../../types' +import type { + ExecutionErrorMetadata, + ExecutionStartMetadata, + ExecutionSuccessMetadata, + PageViewMetadata, + TelemetryProvider +} from '../../types' + +interface DatadogRumInternalContext { + view?: { id?: string } +} interface DatadogRumClient { addAction(name: string, context?: Record): void - setViewContextProperty(name: string, value: unknown): void + getInternalContext(startTime?: number): DatadogRumInternalContext | undefined + setViewName(name: string): void } interface WindowWithDatadogRum extends Window { @@ -10,11 +21,10 @@ interface WindowWithDatadogRum extends Window { } const WORKFLOW_CONTEXT = { - product: 'cloud_generation', - product_surface: 'workspace' + product: 'cloud_generation' } as const -type ProductSurface = +type ViewName = | 'account_access' | 'oauth_consent' | 'support_recovery' @@ -30,14 +40,7 @@ function getDatadogRum(): DatadogRumClient | undefined { return (window as WindowWithDatadogRum).DD_RUM } -function getNavigationType(): NavigationTimingType | 'unknown' { - const [navigation] = performance.getEntriesByType( - 'navigation' - ) as PerformanceNavigationTiming[] - return navigation?.type ?? 'unknown' -} - -function getProductSurface(path = window.location.href): ProductSurface { +function getViewName(path = window.location.href): ViewName { const pathname = new URL(path, window.location.origin).pathname.replace( /\/$/, '' @@ -50,36 +53,43 @@ function getProductSurface(path = window.location.href): ProductSurface { } export class DatadogRumTelemetryProvider implements TelemetryProvider { - constructor() { - getDatadogRum()?.setViewContextProperty( - 'navigation_type', - getNavigationType() - ) - } + private readonly originViewIdsByJobId = new Map() trackPageView(_pageName: string, properties?: PageViewMetadata): void { - getDatadogRum()?.setViewContextProperty( - 'product_surface', - getProductSurface(properties?.path) - ) + getDatadogRum()?.setViewName(getViewName(properties?.path)) } - trackWorkflowExecution(): void { - getDatadogRum()?.addAction('workflow_execution_started', WORKFLOW_CONTEXT) + trackExecutionStarted(metadata: ExecutionStartMetadata): void { + const rum = getDatadogRum() + const originViewId = rum?.getInternalContext(metadata.startTime)?.view?.id + if (originViewId) + this.originViewIdsByJobId.set(metadata.jobId, originViewId) + + rum?.addAction('workflow_execution_started', { + ...WORKFLOW_CONTEXT, + ...(originViewId && { origin_view_id: originViewId }) + }) } - trackExecutionSuccess(): void { - this.trackTerminalOutcome('success') + trackExecutionSuccess(metadata: ExecutionSuccessMetadata): void { + this.trackTerminalOutcome(metadata.jobId, 'success') } - trackExecutionError(): void { - this.trackTerminalOutcome('failure') + trackExecutionError(metadata: ExecutionErrorMetadata): void { + this.trackTerminalOutcome(metadata.jobId, 'failure') } - private trackTerminalOutcome(outcome: 'success' | 'failure'): void { + private trackTerminalOutcome( + jobId: string, + outcome: 'success' | 'failure' + ): void { + const originViewId = this.originViewIdsByJobId.get(jobId) + this.originViewIdsByJobId.delete(jobId) + getDatadogRum()?.addAction('workflow_execution_completed', { ...WORKFLOW_CONTEXT, - outcome + outcome, + ...(originViewId && { origin_view_id: originViewId }) }) } } diff --git a/src/platform/telemetry/types.ts b/src/platform/telemetry/types.ts index 17f9cfeff3..c0ec2fd48e 100644 --- a/src/platform/telemetry/types.ts +++ b/src/platform/telemetry/types.ts @@ -156,6 +156,11 @@ export interface ExecutionErrorMetadata { error?: string } +export interface ExecutionStartMetadata { + jobId: string + startTime: number +} + /** * Execution success metadata */ @@ -632,6 +637,7 @@ export interface TelemetryProvider { // Workflow execution events trackWorkflowExecution?(): void + trackExecutionStarted?(metadata: ExecutionStartMetadata): void trackExecutionError?(metadata: ExecutionErrorMetadata): void trackExecutionSuccess?(metadata: ExecutionSuccessMetadata): void trackSharedWorkflowRun?(metadata: SharedWorkflowRunMetadata): void diff --git a/src/stores/executionStore.test.ts b/src/stores/executionStore.test.ts index fac8c4b149..43a7af9df9 100644 --- a/src/stores/executionStore.test.ts +++ b/src/stores/executionStore.test.ts @@ -20,6 +20,7 @@ const { mockActiveWorkflow, mockOpenWorkflows, mockShowTextPreview, + mockTrackExecutionStarted, mockTrackExecutionError, mockTrackExecutionSuccess, mockTrackSharedWorkflowRun @@ -32,6 +33,7 @@ const { mockActiveWorkflow: shallowRef<{ path?: string } | null>(null), mockOpenWorkflows: shallowRef<{ path: string }[]>([]), mockShowTextPreview: vi.fn(), + mockTrackExecutionStarted: vi.fn(), mockTrackExecutionError: vi.fn(), mockTrackExecutionSuccess: vi.fn(), mockTrackSharedWorkflowRun: vi.fn() @@ -91,6 +93,7 @@ vi.mock('@/platform/distribution/types', async () => ({ vi.mock('@/platform/telemetry', () => ({ useTelemetry: () => ({ + trackExecutionStarted: mockTrackExecutionStarted, trackExecutionError: mockTrackExecutionError, trackExecutionSuccess: mockTrackExecutionSuccess, trackSharedWorkflowRun: mockTrackSharedWorkflowRun @@ -631,6 +634,10 @@ describe('useExecutionStore - workflowStatus', () => { callStoreJob('job-1', workflowA) fireExecutionStart('job-1') + expect(mockTrackExecutionStarted).toHaveBeenCalledWith({ + jobId: 'job-1', + startTime: expect.any(Number) + }) expect(store.getWorkflowStatus(workflowA)).toBe('running') }) diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index 3696130e0b..11f22909f9 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -748,6 +748,10 @@ export const useExecutionStore = defineStore('execution', () => { if (workflow?.path) { ensureSessionWorkflowPath(id, workflow.path) } + useTelemetry()?.trackExecutionStarted({ + jobId: id, + startTime: performance.now() + }) flushPendingWorkflowStatus(String(id), workflow) }