feat: add telemetry to answer for user failed to find template (#6489)

## Summary

Adds mixpanel telemetry with goal of: "We currently only know when a
user opens a template workflow. But we also want to know if they failed
to find what they want"

Example mixpanel query:

```
app:template_library_closed
  WHERE template_selected = false AND time_spent_seconds >= 10
  ```

But can drill down further into what filters were selected etc to answer what they were looking for but couldn't find.

```
 1. Event: app:template_filter_changed
  2. Filter:
- Add formula: "Where user also triggered app:template_library_closed
with template_selected = false in same session"
  3. Breakdown by: search_query
  4. Sort by: Total Count (descending)

  Search Query           Failed Sessions
  -----------------------------------
  "flux video"             45 times
  "sdxl controlnet"     32 times
  "upscaler"               28 times
  (empty/just filter)   20 times
```

```
 Event: app:template_filter_changed
  WHERE filtered_count = 0
    AND user did app:template_library_closed
    with template_selected = false

  Breakdown by: search_query
  ```
  etc.

https://www.notion.so/comfy-org/Number-of-users-who-open-the-template-library-and-where-do-they-click-29b6d73d36508044a595c0fb653ca6dc?source=copy_link

┆Issue is synchronized with this [Notion
page](https://www.notion.so/PR-6489-feat-add-telemetry-to-answer-for-user-failed-to-find-template-29d6d73d365081cdad72fd7c6ada5dc7)
by [Unito](https://www.unito.io)
This commit is contained in:
--list
2025-10-31 16:39:48 -07:00
parent 6dbe00d47c
commit 7a7e1d58a2
6 changed files with 192 additions and 8 deletions

View File

@@ -5,12 +5,20 @@ import type {
ExecutionContext,
ExecutionErrorMetadata,
ExecutionSuccessMetadata,
NodeSearchMetadata,
NodeSearchResultMetadata,
PageVisibilityMetadata,
RunButtonProperties,
SurveyResponses,
TabCountMetadata,
TelemetryEventName,
TelemetryEventProperties,
TelemetryProvider,
TemplateMetadata
TemplateFilterMetadata,
TemplateLibraryClosedMetadata,
TemplateLibraryMetadata,
TemplateMetadata,
WorkflowImportMetadata
} from '../../types'
import { TelemetryEvents } from '../../types'
@@ -351,6 +359,38 @@ export class MixpanelTelemetryProvider implements TelemetryProvider {
this.trackEvent(TelemetryEvents.TEMPLATE_WORKFLOW_OPENED, metadata)
}
trackTemplateLibraryOpened(metadata: TemplateLibraryMetadata): void {
this.trackEvent(TelemetryEvents.TEMPLATE_LIBRARY_OPENED, metadata)
}
trackTemplateLibraryClosed(metadata: TemplateLibraryClosedMetadata): void {
this.trackEvent(TelemetryEvents.TEMPLATE_LIBRARY_CLOSED, metadata)
}
trackWorkflowImported(metadata: WorkflowImportMetadata): void {
this.trackEvent(TelemetryEvents.WORKFLOW_IMPORTED, metadata)
}
trackPageVisibilityChanged(metadata: PageVisibilityMetadata): void {
this.trackEvent(TelemetryEvents.PAGE_VISIBILITY_CHANGED, metadata)
}
trackTabCount(metadata: TabCountMetadata): void {
this.trackEvent(TelemetryEvents.TAB_COUNT_TRACKING, metadata)
}
trackNodeSearch(metadata: NodeSearchMetadata): void {
this.trackEvent(TelemetryEvents.NODE_SEARCH, metadata)
}
trackNodeSearchResultSelected(metadata: NodeSearchResultMetadata): void {
this.trackEvent(TelemetryEvents.NODE_SEARCH_RESULT_SELECTED, metadata)
}
trackTemplateFilterChanged(metadata: TemplateFilterMetadata): void {
this.trackEvent(TelemetryEvents.TEMPLATE_FILTER_CHANGED, metadata)
}
trackWorkflowExecution(): void {
if (this.isOnboardingMode) {
// During onboarding, track basic execution without workflow context

View File

@@ -89,6 +89,76 @@ export interface TemplateMetadata {
template_license?: string
}
/**
* Workflow import metadata
*/
export interface WorkflowImportMetadata {
missing_node_count: number
missing_node_types: string[]
}
/**
* Template library metadata
*/
export interface TemplateLibraryMetadata {
source: 'sidebar' | 'menu' | 'command'
}
/**
* Template library closed metadata
*/
export interface TemplateLibraryClosedMetadata {
template_selected: boolean
time_spent_seconds: number
}
/**
* Page visibility metadata
*/
export interface PageVisibilityMetadata {
visibility_state: 'visible' | 'hidden'
}
/**
* Tab count metadata
*/
export interface TabCountMetadata {
tab_count: number
}
/**
* Node search metadata
*/
export interface NodeSearchMetadata {
query: string
}
/**
* Node search result selection metadata
*/
export interface NodeSearchResultMetadata {
node_type: string
last_query: string
}
/**
* Template filter tracking metadata
*/
export interface TemplateFilterMetadata {
search_query?: string
selected_models: string[]
selected_use_cases: string[]
selected_licenses: string[]
sort_by:
| 'default'
| 'alphabetical'
| 'newest'
| 'vram-low-to-high'
| 'model-size-low-to-high'
filtered_count: number
total_count: number
}
/**
* Core telemetry provider interface
*/
@@ -106,6 +176,24 @@ export interface TelemetryProvider {
// Template workflow events
trackTemplate(metadata: TemplateMetadata): void
trackTemplateLibraryOpened(metadata: TemplateLibraryMetadata): void
trackTemplateLibraryClosed(metadata: TemplateLibraryClosedMetadata): void
// Workflow management events
trackWorkflowImported(metadata: WorkflowImportMetadata): void
// Page visibility events
trackPageVisibilityChanged(metadata: PageVisibilityMetadata): void
// Tab tracking events
trackTabCount(metadata: TabCountMetadata): void
// Node search analytics events
trackNodeSearch(metadata: NodeSearchMetadata): void
trackNodeSearchResultSelected(metadata: NodeSearchResultMetadata): void
// Template filter tracking events
trackTemplateFilterChanged(metadata: TemplateFilterMetadata): void
// Workflow execution events
trackWorkflowExecution(): void
@@ -140,6 +228,24 @@ export const TelemetryEvents = {
// Template Tracking
TEMPLATE_WORKFLOW_OPENED: 'app:template_workflow_opened',
TEMPLATE_LIBRARY_OPENED: 'app:template_library_opened',
TEMPLATE_LIBRARY_CLOSED: 'app:template_library_closed',
// Workflow Management
WORKFLOW_IMPORTED: 'app:workflow_imported',
// Page Visibility
PAGE_VISIBILITY_CHANGED: 'app:page_visibility_changed',
// Tab Tracking
TAB_COUNT_TRACKING: 'app:tab_count_tracking',
// Node Search Analytics
NODE_SEARCH: 'app:node_search',
NODE_SEARCH_RESULT_SELECTED: 'app:node_search_result_selected',
// Template Filter Analytics
TEMPLATE_FILTER_CHANGED: 'app:template_filter_changed',
// Execution Lifecycle
EXECUTION_START: 'execution_start',
@@ -161,3 +267,11 @@ export type TelemetryEventProperties =
| RunButtonProperties
| ExecutionErrorMetadata
| ExecutionSuccessMetadata
| WorkflowImportMetadata
| TemplateLibraryMetadata
| TemplateLibraryClosedMetadata
| PageVisibilityMetadata
| TabCountMetadata
| NodeSearchMetadata
| NodeSearchResultMetadata
| TemplateFilterMetadata