record audio node support (#4289)

Co-authored-by: github-actions <github-actions@github.com>
Co-authored-by: Christian Byrne <cbyrne@comfy.org>

src/extensions/core/uploadAudio.ts

@@ -1,5 +1,9 @@
 import type { LGraphNode } from '@comfyorg/litegraph'
-import type { IStringWidget } from '@comfyorg/litegraph/dist/types/widgets'
+import type {
+  IBaseWidget,
+  IStringWidget
+} from '@comfyorg/litegraph/dist/types/widgets'
+import { MediaRecorder as ExtendableMediaRecorder } from 'extendable-media-recorder'
 
 import { useChainCallback } from '@/composables/functional/useChainCallback'
 import { useNodeDragAndDrop } from '@/composables/node/useNodeDragAndDrop'
@@ -9,6 +13,7 @@ import { t } from '@/i18n'
 import type { ResultItemType } from '@/schemas/apiSchema'
 import type { ComfyNodeDef } from '@/schemas/nodeDefSchema'
 import type { DOMWidget } from '@/scripts/domWidget'
+import { useAudioService } from '@/services/audioService'
 import { useToastStore } from '@/stores/toastStore'
 import { NodeLocatorId } from '@/types'
 import { getNodeByLocatorId } from '@/utils/graphTraversalUtil'
@@ -257,3 +262,167 @@ app.registerExtension({
     }
   }
 })
+
+app.registerExtension({
+  name: 'Comfy.RecordAudio',
+
+  getCustomWidgets() {
+    return {
+      AUDIO_RECORD(node, inputName: string) {
+        const audio = document.createElement('audio')
+        audio.controls = true
+        audio.classList.add('comfy-audio')
+        audio.setAttribute('name', 'media')
+
+        const audioUIWidget: DOMWidget<HTMLAudioElement, string> =
+          node.addDOMWidget(inputName, /* name=*/ 'audioUI', audio)
+
+        let mediaRecorder: MediaRecorder | null = null
+        let isRecording = false
+        let audioChunks: Blob[] = []
+        let currentStream: MediaStream | null = null
+        let recordWidget: IBaseWidget | null = null
+
+        let stopPromise: Promise<void> | null = null
+        let stopResolve: (() => void) | null = null
+
+        audioUIWidget.serializeValue = async () => {
+          if (isRecording && mediaRecorder) {
+            stopPromise = new Promise((resolve) => {
+              stopResolve = resolve
+            })
+
+            mediaRecorder.stop()
+
+            await stopPromise
+          }
+
+          const audioSrc = audioUIWidget.element.src
+
+          if (!audioSrc) {
+            useToastStore().addAlert(t('g.noAudioRecorded'))
+            return ''
+          }
+
+          const blob = await fetch(audioSrc).then((r) => r.blob())
+
+          return await useAudioService().convertBlobToFileAndSubmit(blob)
+        }
+
+        recordWidget = node.addWidget(
+          'button',
+          inputName,
+          '',
+          async () => {
+            if (!isRecording) {
+              try {
+                currentStream = await navigator.mediaDevices.getUserMedia({
+                  audio: true
+                })
+
+                mediaRecorder = new ExtendableMediaRecorder(currentStream, {
+                  mimeType: 'audio/wav'
+                }) as unknown as MediaRecorder
+
+                audioChunks = []
+
+                mediaRecorder.ondataavailable = (event) => {
+                  audioChunks.push(event.data)
+                }
+
+                mediaRecorder.onstop = async () => {
+                  const audioBlob = new Blob(audioChunks, { type: 'audio/wav' })
+
+                  useAudioService().stopAllTracks(currentStream)
+
+                  if (
+                    audioUIWidget.element.src &&
+                    audioUIWidget.element.src.startsWith('blob:')
+                  ) {
+                    URL.revokeObjectURL(audioUIWidget.element.src)
+                  }
+
+                  audioUIWidget.element.src = URL.createObjectURL(audioBlob)
+
+                  isRecording = false
+
+                  if (recordWidget) {
+                    recordWidget.label = t('g.startRecording')
+                  }
+
+                  if (stopResolve) {
+                    stopResolve()
+                    stopResolve = null
+                    stopPromise = null
+                  }
+                }
+
+                mediaRecorder.onerror = (event) => {
+                  console.error('MediaRecorder error:', event)
+                  useAudioService().stopAllTracks(currentStream)
+                  isRecording = false
+
+                  if (recordWidget) {
+                    recordWidget.label = t('g.startRecording')
+                  }
+
+                  if (stopResolve) {
+                    stopResolve()
+                    stopResolve = null
+                    stopPromise = null
+                  }
+                }
+
+                mediaRecorder.start()
+                isRecording = true
+                if (recordWidget) {
+                  recordWidget.label = t('g.stopRecording')
+                }
+              } catch (err) {
+                console.error('Error accessing microphone:', err)
+                useToastStore().addAlert(t('g.micPermissionDenied'))
+
+                if (mediaRecorder) {
+                  try {
+                    mediaRecorder.stop()
+                  } catch {}
+                }
+                useAudioService().stopAllTracks(currentStream)
+                currentStream = null
+                isRecording = false
+                if (recordWidget) {
+                  recordWidget.label = t('g.startRecording')
+                }
+              }
+            } else if (mediaRecorder && isRecording) {
+              mediaRecorder.stop()
+            }
+          },
+          { serialize: false }
+        )
+
+        recordWidget.label = t('g.startRecording')
+
+        const originalOnRemoved = node.onRemoved
+        node.onRemoved = function () {
+          if (isRecording && mediaRecorder) {
+            mediaRecorder.stop()
+          }
+          useAudioService().stopAllTracks(currentStream)
+          if (audioUIWidget.element.src?.startsWith('blob:')) {
+            URL.revokeObjectURL(audioUIWidget.element.src)
+          }
+          originalOnRemoved?.call(this)
+        }
+
+        return { widget: recordWidget }
+      }
+    }
+  },
+
+  async nodeCreated(node) {
+    if (node.constructor.comfyClass !== 'RecordAudio') return
+
+    await useAudioService().registerWavEncoder()
+  }
+})