From 91b5a7de17c060f28e24f436524e9604bc665a32 Mon Sep 17 00:00:00 2001 From: Benjamin Lu Date: Fri, 31 Oct 2025 12:55:00 -0700 Subject: [PATCH] feat(telemetry): track total node count, subgraphs, and API-node details in RUN_BUTTON_CLICKED (#6468) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary - Add richer run-button telemetry: total node count, subgraph count, whether API nodes are present, and unique API node names. - Add provider method/event for “Add API credit” button clicks. - Include a one-off script to normalize Mixpanel survey properties (industry/useCase) for better analytics. Changes - Types: extend telemetry payloads - RunButtonProperties adds total_node_count, subgraph_count, has_api_nodes, api_node_names (src/platform/telemetry/types.ts:42) - ExecutionContext adds same fields (src/platform/telemetry/types.ts:61) - New event + provider API: ADD_API_CREDIT_BUTTON_CLICKED, trackAddApiCreditButtonClicked() (src/platform/telemetry/types.ts:173, src/platform/telemetry/types.ts:180) - Mixpanel provider - Compute node metrics in a single graph traversal and include them in RUN_BUTTON_CLICKED (src/platform/telemetry/providers/cloud/MixpanelTelemetryProvider.ts:169) - Fields populated: total_node_count, subgraph_count, has_api_nodes, api_node_names (src/platform/telemetry/providers/cloud/MixpanelTelemetryProvider.ts:177) - Add trackAddApiCreditButtonClicked() implementation (src/platform/telemetry/providers/cloud/MixpanelTelemetryProvider.ts:152) - Script - Add scripts/survey-data-migration.ts to normalize industry/useCase user properties using existing survey normalization utils; includes a simulation and a production outline (scripts/survey-data-migration.ts:1) Motivation - Improves insight into workflow complexity and API-node adoption directly at run time. - Normalizes free-text survey fields to reduce category proliferation and improve reporting quality. Validation - Run a workflow with/without API nodes and subgraphs; confirm telemetry includes: - total_node_count, subgraph_count, has_api_nodes, api_node_names - Click “Add API credit” and confirm app:add_api_credit_button_clicked is sent. - No user-visible changes; telemetry only runs in cloud builds. Impact - Telemetry payload shape expands; backend ingestion should accept the new properties. - Metrics computed in a single pass over the graph for efficiency. --------- Co-authored-by: bymyself --- .../cloud/MixpanelTelemetryProvider.ts | 58 +++++++++++++++---- src/platform/telemetry/types.ts | 9 +++ 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/platform/telemetry/providers/cloud/MixpanelTelemetryProvider.ts b/src/platform/telemetry/providers/cloud/MixpanelTelemetryProvider.ts index b15e7948e1..b09041dab2 100644 --- a/src/platform/telemetry/providers/cloud/MixpanelTelemetryProvider.ts +++ b/src/platform/telemetry/providers/cloud/MixpanelTelemetryProvider.ts @@ -149,6 +149,10 @@ export class MixpanelTelemetryProvider implements TelemetryProvider { this.trackEvent(eventName) } + trackAddApiCreditButtonClicked(): void { + this.trackEvent(TelemetryEvents.ADD_API_CREDIT_BUTTON_CLICKED) + } + trackMonthlySubscriptionSucceeded(): void { this.trackEvent(TelemetryEvents.MONTHLY_SUBSCRIPTION_SUCCEEDED) } @@ -169,7 +173,11 @@ export class MixpanelTelemetryProvider implements TelemetryProvider { const runButtonProperties: RunButtonProperties = { subscribe_to_run: options?.subscribe_to_run || false, workflow_type: executionContext.is_template ? 'template' : 'custom', - workflow_name: executionContext.workflow_name ?? 'untitled' + workflow_name: executionContext.workflow_name ?? 'untitled', + total_node_count: executionContext.total_node_count, + subgraph_count: executionContext.subgraph_count, + has_api_nodes: executionContext.has_api_nodes, + api_node_names: executionContext.api_node_names } this.trackEvent(TelemetryEvents.RUN_BUTTON_CLICKED, runButtonProperties) @@ -271,22 +279,50 @@ export class MixpanelTelemetryProvider implements TelemetryProvider { const activeWorkflow = workflowStore.activeWorkflow // Calculate node metrics in a single traversal - const nodeMetrics = reduceAllNodes( + type NodeMetrics = { + custom_node_count: number + api_node_count: number + subgraph_count: number + total_node_count: number + has_api_nodes: boolean + api_node_names: string[] + } + + const nodeCounts = reduceAllNodes( app.graph, - (acc, node) => { + (metrics, node) => { const nodeDef = nodeDefStore.nodeDefsByName[node.type] const isCustomNode = nodeDef?.nodeSource?.type === NodeSourceType.CustomNodes const isApiNode = nodeDef?.api_node === true const isSubgraph = node.isSubgraphNode?.() === true - return { - custom_node_count: acc.custom_node_count + (isCustomNode ? 1 : 0), - api_node_count: acc.api_node_count + (isApiNode ? 1 : 0), - subgraph_count: acc.subgraph_count + (isSubgraph ? 1 : 0) + if (isApiNode) { + metrics.has_api_nodes = true + const canonicalName = nodeDef?.name + if ( + canonicalName && + !metrics.api_node_names.includes(canonicalName) + ) { + metrics.api_node_names.push(canonicalName) + } } + + metrics.custom_node_count += isCustomNode ? 1 : 0 + metrics.api_node_count += isApiNode ? 1 : 0 + metrics.subgraph_count += isSubgraph ? 1 : 0 + metrics.total_node_count += 1 + + return metrics }, - { custom_node_count: 0, api_node_count: 0, subgraph_count: 0 } + { + custom_node_count: 0, + api_node_count: 0, + subgraph_count: 0, + total_node_count: 0, + has_api_nodes: false, + api_node_names: [] + } ) if (activeWorkflow?.filename) { @@ -312,21 +348,21 @@ export class MixpanelTelemetryProvider implements TelemetryProvider { template_models: englishMetadata?.models ?? template?.models, template_use_case: englishMetadata?.useCase ?? template?.useCase, template_license: englishMetadata?.license ?? template?.license, - ...nodeMetrics + ...nodeCounts } } return { is_template: false, workflow_name: activeWorkflow.filename, - ...nodeMetrics + ...nodeCounts } } return { is_template: false, workflow_name: undefined, - ...nodeMetrics + ...nodeCounts } } } diff --git a/src/platform/telemetry/types.ts b/src/platform/telemetry/types.ts index 2dbf10c2a6..71df735d2e 100644 --- a/src/platform/telemetry/types.ts +++ b/src/platform/telemetry/types.ts @@ -42,6 +42,10 @@ export interface RunButtonProperties { subscribe_to_run: boolean workflow_type: 'template' | 'custom' workflow_name: string + total_node_count: number + subgraph_count: number + has_api_nodes: boolean + api_node_names: string[] } /** @@ -61,6 +65,9 @@ export interface ExecutionContext { custom_node_count: number api_node_count: number subgraph_count: number + total_node_count: number + has_api_nodes: boolean + api_node_names: string[] } /** @@ -173,6 +180,7 @@ export interface TelemetryProvider { // Subscription flow events trackSubscription(event: 'modal_opened' | 'subscribe_clicked'): void trackMonthlySubscriptionSucceeded(): void + trackAddApiCreditButtonClicked(): void trackApiCreditTopupButtonPurchaseClicked(amount: number): void trackRunButton(options?: { subscribe_to_run?: boolean }): void @@ -225,6 +233,7 @@ export const TelemetryEvents = { SUBSCRIPTION_REQUIRED_MODAL_OPENED: 'app:subscription_required_modal_opened', SUBSCRIBE_NOW_BUTTON_CLICKED: 'app:subscribe_now_button_clicked', MONTHLY_SUBSCRIPTION_SUCCEEDED: 'app:monthly_subscription_succeeded', + ADD_API_CREDIT_BUTTON_CLICKED: 'app:add_api_credit_button_clicked', API_CREDIT_TOPUP_BUTTON_PURCHASE_CLICKED: 'app:api_credit_topup_button_purchase_clicked',