From 8ba98bad073b57f539405ff444ddaa267163f9fe Mon Sep 17 00:00:00 2001 From: Johnpaul Date: Thu, 25 Sep 2025 23:02:15 +0100 Subject: [PATCH] [feat] Add widget factories for Vue custom widgets system --- .../core/factories/AudioWidgetFactory.ts | 44 ++ .../factories/LiteGraphAudioWidgetFactory.ts | 446 ++++++++++++++++++ .../core/factories/VueAudioWidgetFactory.ts | 145 ++++++ 3 files changed, 635 insertions(+) create mode 100644 src/extensions/core/factories/AudioWidgetFactory.ts create mode 100644 src/extensions/core/factories/LiteGraphAudioWidgetFactory.ts create mode 100644 src/extensions/core/factories/VueAudioWidgetFactory.ts diff --git a/src/extensions/core/factories/AudioWidgetFactory.ts b/src/extensions/core/factories/AudioWidgetFactory.ts new file mode 100644 index 0000000000..8ee167480a --- /dev/null +++ b/src/extensions/core/factories/AudioWidgetFactory.ts @@ -0,0 +1,44 @@ +/** + * Factory interface for creating audio widgets + * Provides consistent API for both Vue and LiteGraph implementations + */ +import type { LGraphNode } from '@/lib/litegraph/src/litegraph' +import type { IBaseWidget } from '@/lib/litegraph/src/types/widgets' + +export interface WidgetCreationResult { + widget: IBaseWidget + minWidth?: number + minHeight?: number +} + +export interface AudioWidgetFactory { + /** + * Creates an AUDIO_UI widget for audio playback/preview + */ + createAudioUI(node: LGraphNode, inputName: string): WidgetCreationResult + + /** + * Creates an AUDIOUPLOAD widget for file uploads + */ + createAudioUpload(node: LGraphNode, inputName: string): WidgetCreationResult + + /** + * Creates an AUDIO_RECORD widget for audio recording + */ + createAudioRecord(node: LGraphNode, inputName: string): WidgetCreationResult + + /** + * Hook called before registering node definitions + */ + beforeRegisterNodeDef?(nodeType: any, nodeData: any): void + + /** + * Hook called when node outputs are updated + */ + onNodeOutputsUpdated?(nodeOutputs: Record): void + + /** + * Hook called when a node is created (for RecordAudio setup) + */ + onNodeCreated?(node: LGraphNode): Promise +} diff --git a/src/extensions/core/factories/LiteGraphAudioWidgetFactory.ts b/src/extensions/core/factories/LiteGraphAudioWidgetFactory.ts new file mode 100644 index 0000000000..891511cc13 --- /dev/null +++ b/src/extensions/core/factories/LiteGraphAudioWidgetFactory.ts @@ -0,0 +1,446 @@ +/** + * LiteGraph Audio Widget Factory + * Creates full DOM-based audio widgets with rich functionality + */ +import { MediaRecorder as ExtendableMediaRecorder } from 'extendable-media-recorder' + +import { useChainCallback } from '@/composables/functional/useChainCallback' +import { useNodeDragAndDrop } from '@/composables/node/useNodeDragAndDrop' +import { useNodeFileInput } from '@/composables/node/useNodeFileInput' +import { useNodePaste } from '@/composables/node/useNodePaste' +import { t } from '@/i18n' +import { type LGraphNode } from '@/lib/litegraph/src/litegraph' +import type { + IBaseWidget, + IStringWidget +} from '@/lib/litegraph/src/types/widgets' +import { useToastStore } from '@/platform/updates/common/toastStore' +import type { DOMWidget } from '@/scripts/domWidget' +import { useAudioService } from '@/services/audioService' +import { vueWidgetSerializationStore } from '@/stores/vueWidgetSerializationStore' +import { getNodeByLocatorId } from '@/utils/graphTraversalUtil' + +import { api } from '../../../scripts/api' +import { app } from '../../../scripts/app' +import type { + AudioWidgetFactory, + WidgetCreationResult +} from './AudioWidgetFactory' + +export class LiteGraphAudioWidgetFactory implements AudioWidgetFactory { + beforeRegisterNodeDef(nodeType: any, nodeData: any): void { + // Add audioUI widget requirement for audio nodes + if ( + [ + 'LoadAudio', + 'SaveAudio', + 'PreviewAudio', + 'SaveAudioMP3', + 'SaveAudioOpus', + 'RecordAudio' + ].includes(nodeType.prototype?.comfyClass) + ) { + nodeData.input.required.audioUI = ['AUDIO_UI', {}] + } + + // Add upload widget requirement for nodes with audio_upload flag + if (nodeData?.input?.required?.audio?.[1]?.audio_upload === true) { + nodeData.input.required.upload = ['AUDIOUPLOAD', {}] + } + } + + createAudioUI(node: LGraphNode, inputName: string): WidgetCreationResult { + // Create DOM audio element + const audio = document.createElement('audio') + audio.controls = true + audio.classList.add('comfy-audio') + audio.setAttribute('name', 'media') + + const audioUIWidget: DOMWidget = + node.addDOMWidget(inputName, 'audioUI', audio) + audioUIWidget.serialize = false + + const { nodeData } = node.constructor + if (nodeData == null) throw new TypeError('nodeData is null') + + const isOutputNode = nodeData.output_node + if (isOutputNode) { + // Hide the audio widget when there is no audio initially + audioUIWidget.element.classList.add('empty-audio-widget') + + // Populate the audio widget UI on node execution + const onExecuted = node.onExecuted + const factory = this // Capture factory reference for callback + node.onExecuted = function (message: any) { + // @ts-expect-error fixme ts strict error + onExecuted?.apply(this, arguments) + const audios = message.audio + if (!audios) return + const audio = audios[0] + audioUIWidget.element.src = api.apiURL( + factory.getResourceURL(audio.subfolder, audio.filename, audio.type) + ) + audioUIWidget.element.classList.remove('empty-audio-widget') + } + } + + audioUIWidget.onRemove = useChainCallback(audioUIWidget.onRemove, () => { + if (!audioUIWidget.element) return + audioUIWidget.element.pause() + audioUIWidget.element.src = '' + audioUIWidget.element.remove() + }) + + // Add serialization support for RecordAudio nodes + if ((node.constructor as any).comfyClass === 'RecordAudio') { + const nodeId = node.id + audioUIWidget.serializeValue = async () => { + let serializationFn = vueWidgetSerializationStore.get( + `${nodeId}-audioUI` + ) + + // Fallback: try with current node.id in case it changed + if (!serializationFn && node.id !== nodeId) { + serializationFn = vueWidgetSerializationStore.get( + `${node.id}-audioUI` + ) + } + + if (serializationFn) { + const result = await serializationFn() + + // Update both LiteGraph widgets for consistency + const audioWidget = node.widgets?.find((w) => w.name === 'audio') + const audioUIWidget = node.widgets?.find((w) => w.name === 'audioUI') + + if (audioWidget && result) { + audioWidget.value = result + } + if (audioUIWidget && result) { + audioUIWidget.value = result + } + + return result + } + + return '' + } + } + + return { widget: audioUIWidget } + } + + createAudioUpload(node: LGraphNode, inputName: string): WidgetCreationResult { + // Find the related audio widgets + const audioWidget = node.widgets?.find( + (w) => w.name === 'audio' + ) as IStringWidget + const audioUIWidget = node.widgets?.find( + (w) => w.name === 'audioUI' + ) as unknown as DOMWidget + + const factory = this // Capture factory reference + const onAudioWidgetUpdate = () => { + audioUIWidget.element.src = api.apiURL( + factory.getResourceURL( + ...factory.splitFilePath(audioWidget.value as string) + ) + ) + } + + // Initially load default audio file to audioUIWidget + if (audioWidget.value) { + onAudioWidgetUpdate() + } + audioWidget.callback = onAudioWidgetUpdate + + // Load saved audio file widget values if restoring from workflow + const onGraphConfigured = node.onGraphConfigured + node.onGraphConfigured = function () { + // @ts-expect-error fixme ts strict error + onGraphConfigured?.apply(this, arguments) + if (audioWidget.value) { + onAudioWidgetUpdate() + } + } + + const handleUpload = async (files: File[]) => { + if (files?.length) { + await this.uploadFile(audioWidget, audioUIWidget, files[0], true) + } + return files + } + + const isAudioFile = (file: File) => file.type.startsWith('audio/') + + const { openFileSelection } = useNodeFileInput(node, { + accept: 'audio/*', + onSelect: handleUpload + }) + + // The widget to pop up the upload dialog + const uploadWidget = node.addWidget( + 'button', + inputName, + '', + openFileSelection, + { + serialize: false + } + ) + uploadWidget.label = t('g.choose_file_to_upload') + + useNodeDragAndDrop(node, { + fileFilter: isAudioFile, + onDrop: handleUpload + }) + + useNodePaste(node, { + fileFilter: isAudioFile, + onPaste: handleUpload + }) + + node.previewMediaType = 'audio' + + return { widget: uploadWidget } + } + + createAudioRecord(node: LGraphNode, inputName: string): WidgetCreationResult { + const audio = document.createElement('audio') + audio.controls = true + audio.classList.add('comfy-audio') + audio.setAttribute('name', 'media') + + const audioUIWidget: DOMWidget = + node.addDOMWidget(inputName, 'audioUI', audio) + + let mediaRecorder: MediaRecorder | null = null + let isRecording = false + let audioChunks: Blob[] = [] + let currentStream: MediaStream | null = null + let recordWidget: IBaseWidget | null = null + + let stopPromise: Promise | null = null + let stopResolve: (() => void) | null = null + + audioUIWidget.serializeValue = async () => { + if (isRecording && mediaRecorder) { + stopPromise = new Promise((resolve) => { + stopResolve = resolve + }) + + mediaRecorder.stop() + await stopPromise + } + + const audioSrc = audioUIWidget.element.src + + if (!audioSrc) { + useToastStore().addAlert(t('g.noAudioRecorded')) + return '' + } + + const blob = await fetch(audioSrc).then((r) => r.blob()) + return await useAudioService().convertBlobToFileAndSubmit(blob) + } + + recordWidget = node.addWidget( + 'button', + inputName, + '', + async () => { + if (!isRecording) { + try { + currentStream = await navigator.mediaDevices.getUserMedia({ + audio: true + }) + + mediaRecorder = new ExtendableMediaRecorder(currentStream, { + mimeType: 'audio/wav' + }) as unknown as MediaRecorder + + audioChunks = [] + + mediaRecorder.ondataavailable = (event) => { + audioChunks.push(event.data) + } + + mediaRecorder.onstop = async () => { + const audioBlob = new Blob(audioChunks, { type: 'audio/wav' }) + + useAudioService().stopAllTracks(currentStream) + + if ( + audioUIWidget.element.src && + audioUIWidget.element.src.startsWith('blob:') + ) { + URL.revokeObjectURL(audioUIWidget.element.src) + } + + audioUIWidget.element.src = URL.createObjectURL(audioBlob) + isRecording = false + + if (recordWidget) { + recordWidget.label = t('g.startRecording') + } + + if (stopResolve) { + stopResolve() + stopResolve = null + stopPromise = null + } + } + + mediaRecorder.onerror = (event) => { + console.error('MediaRecorder error:', event) + useAudioService().stopAllTracks(currentStream) + isRecording = false + + if (recordWidget) { + recordWidget.label = t('g.startRecording') + } + + if (stopResolve) { + stopResolve() + stopResolve = null + stopPromise = null + } + } + + mediaRecorder.start() + isRecording = true + if (recordWidget) { + recordWidget.label = t('g.stopRecording') + } + } catch (err) { + console.error('Error accessing microphone:', err) + useToastStore().addAlert(t('g.micPermissionDenied')) + + if (mediaRecorder) { + try { + mediaRecorder.stop() + } catch {} + } + useAudioService().stopAllTracks(currentStream) + currentStream = null + isRecording = false + if (recordWidget) { + recordWidget.label = t('g.startRecording') + } + } + } else if (mediaRecorder && isRecording) { + mediaRecorder.stop() + } + }, + { serialize: false } + ) + + recordWidget.label = t('g.startRecording') + + const originalOnRemoved = node.onRemoved + node.onRemoved = function () { + if (isRecording && mediaRecorder) { + mediaRecorder.stop() + } + useAudioService().stopAllTracks(currentStream) + if (audioUIWidget.element.src?.startsWith('blob:')) { + URL.revokeObjectURL(audioUIWidget.element.src) + } + originalOnRemoved?.call(this) + } + + return { widget: recordWidget } + } + + onNodeOutputsUpdated(nodeOutputs: Record): void { + for (const [nodeLocatorId, output] of Object.entries(nodeOutputs)) { + if ('audio' in output) { + const node = getNodeByLocatorId(app.graph, nodeLocatorId) + if (!node) continue + + const audioUIWidget = node.widgets?.find( + (w) => w.name === 'audioUI' + ) as unknown as DOMWidget + const audio = output.audio[0] + audioUIWidget.element.src = api.apiURL( + this.getResourceURL(audio.subfolder, audio.filename, audio.type) + ) + audioUIWidget.element.classList.remove('empty-audio-widget') + } + } + } + + async onNodeCreated(node: LGraphNode): Promise { + if ((node.constructor as any).comfyClass === 'RecordAudio') { + await useAudioService().registerWavEncoder() + } + } + + // Helper methods + private splitFilePath(path: string): [string, string] { + const folder_separator = path.lastIndexOf('/') + if (folder_separator === -1) { + return ['', path] + } + return [ + path.substring(0, folder_separator), + path.substring(folder_separator + 1) + ] + } + + private getResourceURL( + subfolder: string, + filename: string, + type = 'input' + ): string { + const params = [ + 'filename=' + encodeURIComponent(filename), + 'type=' + type, + 'subfolder=' + subfolder, + app.getRandParam().substring(1) + ].join('&') + return `/view?${params}` + } + + private async uploadFile( + audioWidget: IStringWidget, + audioUIWidget: DOMWidget, + file: File, + updateNode: boolean, + pasted: boolean = false + ): Promise { + try { + const body = new FormData() + body.append('image', file) + if (pasted) body.append('subfolder', 'pasted') + + const resp = await api.fetchApi('/upload/image', { + method: 'POST', + body + }) + + if (resp.status === 200) { + const data = await resp.json() + let path = data.name + if (data.subfolder) path = data.subfolder + '/' + path + + if (!audioWidget.options?.values?.includes(path)) { + audioWidget.options = audioWidget.options || {} + audioWidget.options.values = audioWidget.options.values || [] + audioWidget.options.values.push(path) + } + + if (updateNode) { + audioUIWidget.element.src = api.apiURL( + this.getResourceURL(...this.splitFilePath(path)) + ) + audioWidget.value = path + } + } else { + useToastStore().addAlert(resp.status + ' - ' + resp.statusText) + } + } catch (error) { + useToastStore().addAlert(String(error)) + } + } +} diff --git a/src/extensions/core/factories/VueAudioWidgetFactory.ts b/src/extensions/core/factories/VueAudioWidgetFactory.ts new file mode 100644 index 0000000000..5317ee2446 --- /dev/null +++ b/src/extensions/core/factories/VueAudioWidgetFactory.ts @@ -0,0 +1,145 @@ +/** + * Vue Audio Widget Factory + * Creates minimal placeholder widgets that Vue components can recognize and render + */ +import { t } from '@/i18n' +import { type LGraphNode } from '@/lib/litegraph/src/litegraph' +import type { IBaseWidget } from '@/lib/litegraph/src/types/widgets' +import { useAudioService } from '@/services/audioService' +import { vueWidgetSerializationStore } from '@/stores/vueWidgetSerializationStore' +import { getNodeByLocatorId } from '@/utils/graphTraversalUtil' + +import { api } from '../../../scripts/api' +import { app } from '../../../scripts/app' +import type { + AudioWidgetFactory, + WidgetCreationResult +} from './AudioWidgetFactory' + +export class VueAudioWidgetFactory implements AudioWidgetFactory { + beforeRegisterNodeDef(nodeType: any, nodeData: any): void { + // Add audioUI widget requirement for audio nodes + if ( + [ + 'LoadAudio', + 'SaveAudio', + 'PreviewAudio', + 'SaveAudioMP3', + 'SaveAudioOpus', + 'RecordAudio' + ].includes(nodeType.prototype?.comfyClass) + ) { + nodeData.input.required.audioUI = ['AUDIO_UI', {}] + } + + // Add upload widget requirement for nodes with audio_upload flag + if (nodeData?.input?.required?.audio?.[1]?.audio_upload === true) { + nodeData.input.required.upload = ['AUDIOUPLOAD', {}] + } + } + + createAudioUI(node: LGraphNode, inputName: string): WidgetCreationResult { + // Create simple placeholder widget for Vue + const audioUIWidget = node.addWidget('custom', inputName, '', () => {}, { + serialize: false + }) as IBaseWidget + + // Set the type that Vue components expect + audioUIWidget.type = 'AUDIO_UI' + + // Add serialization support for RecordAudio nodes + if ((node.constructor as any).comfyClass === 'RecordAudio') { + audioUIWidget.serializeValue = async () => { + const serializationFn = vueWidgetSerializationStore.get( + `${node.id}-audioUI` + ) + if (serializationFn) { + const result = await serializationFn() + audioUIWidget.value = result + return result + } + return audioUIWidget.value || '' + } + } + + return { widget: audioUIWidget } + } + + createAudioUpload(node: LGraphNode, inputName: string): WidgetCreationResult { + // Create simple placeholder widget for Vue + const uploadWidget = node.addWidget('button', inputName, '', () => {}, { + serialize: false + }) as IBaseWidget + + // Set the type that Vue components expect + uploadWidget.type = 'AUDIOUPLOAD' + uploadWidget.label = t('g.choose_file_to_upload') + + return { widget: uploadWidget } + } + + createAudioRecord(node: LGraphNode, inputName: string): WidgetCreationResult { + // Create simple placeholder widget for Vue + const recordWidget = node.addWidget('custom', inputName, '', () => {}, { + serialize: true + }) as IBaseWidget + + // Set the type that Vue components expect + recordWidget.type = 'AUDIO_RECORD' + + // Set up serialization bridge for RecordAudio + recordWidget.serializeValue = async () => { + const serializationFn = vueWidgetSerializationStore.get( + `${node.id}-audioUI` + ) + if (serializationFn) { + const result = await serializationFn() + recordWidget.value = result + return result + } + return recordWidget.value || '' + } + + return { widget: recordWidget } + } + + onNodeOutputsUpdated(nodeOutputs: Record): void { + // Helper function to get resource URL + function getResourceURL( + subfolder: string, + filename: string, + type = 'input' + ): string { + const params = [ + 'filename=' + encodeURIComponent(filename), + 'type=' + type, + 'subfolder=' + subfolder, + app.getRandParam().substring(1) + ].join('&') + return `/view?${params}` + } + + for (const [nodeLocatorId, output] of Object.entries(nodeOutputs)) { + if ('audio' in output) { + const node = getNodeByLocatorId(app.graph, nodeLocatorId) + if (!node) continue + + const audioUIWidget = node.widgets?.find( + (w) => w.name === 'audioUI' + ) as IBaseWidget + if (audioUIWidget) { + const audio = output.audio[0] + audioUIWidget.value = api.apiURL( + getResourceURL(audio.subfolder, audio.filename, audio.type) + ) + } + } + } + } + + async onNodeCreated(node: LGraphNode): Promise { + if ((node.constructor as any).comfyClass === 'RecordAudio') { + await useAudioService().registerWavEncoder() + } + } +}