[3d] add recording video support (#3749)

Co-authored-by: github-actions <github-actions@github.com>

src/extensions/core/load3d.ts

@@ -215,6 +215,8 @@ useExtensionService().registerExtension({
         sceneWidget.serializeValue = async () => {
           node.properties['Camera Info'] = load3d.getCameraState()
 
+          load3d.stopRecording()
+
           const {
             scene: imageData,
             mask: maskData,
@@ -234,13 +236,26 @@ useExtensionService().registerExtension({
 
           load3d.handleResize()
 
-          return {
+          const returnVal = {
             image: `threed/${data.name} [temp]`,
             mask: `threed/${dataMask.name} [temp]`,
             normal: `threed/${dataNormal.name} [temp]`,
             lineart: `threed/${dataLineart.name} [temp]`,
-            camera_info: node.properties['Camera Info']
+            camera_info: node.properties['Camera Info'],
+            recording: ''
           }
+
+          const recordingData = load3d.getRecordingData()
+
+          if (recordingData) {
+            const [recording] = await Promise.all([
+              Load3dUtils.uploadTempImage(recordingData, 'recording', 'mp4')
+            ])
+
+            returnVal['recording'] = `threed/${recording.name} [temp]`
+          }
+
+          return returnVal
         }
       }
     }

src/extensions/core/load3d/Load3d.ts

@@ -12,6 +12,7 @@ import { ModelExporter } from './ModelExporter'
 import { ModelManager } from './ModelManager'
 import { NodeStorage } from './NodeStorage'
 import { PreviewManager } from './PreviewManager'
+import { RecordingManager } from './RecordingManager'
 import { SceneManager } from './SceneManager'
 import { ViewHelperManager } from './ViewHelperManager'
 import {
@@ -38,6 +39,7 @@ class Load3d {
   protected previewManager: PreviewManager
   protected loaderManager: LoaderManager
   protected modelManager: ModelManager
+  protected recordingManager: RecordingManager
 
   STATUS_MOUSE_ON_NODE: boolean
   STATUS_MOUSE_ON_SCENE: boolean
@@ -118,6 +120,11 @@ class Load3d {
 
     this.loaderManager = new LoaderManager(this.modelManager, this.eventManager)
 
+    this.recordingManager = new RecordingManager(
+      this.sceneManager.scene,
+      this.renderer,
+      this.eventManager
+    )
     this.sceneManager.init()
     this.cameraManager.init()
     this.controlsManager.init()
@@ -439,7 +446,39 @@ class Load3d {
     return this.nodeStorage.loadNodeProperty(name, defaultValue)
   }
 
-  remove(): void {
+  public async startRecording(): Promise<void> {
+    this.viewHelperManager.visibleViewHelper(false)
+
+    return this.recordingManager.startRecording()
+  }
+
+  public stopRecording(): void {
+    this.viewHelperManager.visibleViewHelper(true)
+
+    this.recordingManager.stopRecording()
+  }
+
+  public isRecording(): boolean {
+    return this.recordingManager.hasRecording()
+  }
+
+  public getRecordingDuration(): number {
+    return this.recordingManager.getRecordingDuration()
+  }
+
+  public getRecordingData(): string | null {
+    return this.recordingManager.getRecordingData()
+  }
+
+  public exportRecording(filename?: string): void {
+    this.recordingManager.exportRecording(filename)
+  }
+
+  public clearRecording(): void {
+    this.recordingManager.clearRecording()
+  }
+
+  public remove(): void {
     if (this.animationFrameId !== null) {
       cancelAnimationFrame(this.animationFrameId)
     }
@@ -452,6 +491,7 @@ class Load3d {
     this.previewManager.dispose()
     this.loaderManager.dispose()
     this.modelManager.dispose()
+    this.recordingManager.dispose()
 
     this.renderer.dispose()
     this.renderer.domElement.remove()

src/extensions/core/load3d/Load3dUtils.ts

@@ -4,10 +4,16 @@ import { app } from '@/scripts/app'
 import { useToastStore } from '@/stores/toastStore'
 
 class Load3dUtils {
-  static async uploadTempImage(imageData: string, prefix: string) {
+  static async uploadTempImage(
+    imageData: string,
+    prefix: string,
+    fileType: string = 'png'
+  ) {
     const blob = await fetch(imageData).then((r) => r.blob())
-    const name = `${prefix}_${Date.now()}.png`
-    const file = new File([blob], name)
+    const name = `${prefix}_${Date.now()}.${fileType}`
+    const file = new File([blob], name, {
+      type: fileType === 'mp4' ? 'video/mp4' : 'image/png'
+    })
 
     const body = new FormData()
     body.append('image', file)
@@ -20,7 +26,7 @@ class Load3dUtils {
     })
 
     if (resp.status !== 200) {
-      const err = `Error uploading temp image: ${resp.status} - ${resp.statusText}`
+      const err = `Error uploading temp file: ${resp.status} - ${resp.statusText}`
       useToastStore().addAlert(err)
       throw new Error(err)
     }

src/extensions/core/load3d/RecordingManager.ts

@@ -0,0 +1,183 @@
+import * as THREE from 'three'
+
+import { EventManagerInterface } from './interfaces'
+
+export class RecordingManager {
+  private mediaRecorder: MediaRecorder | null = null
+  private recordedChunks: Blob[] = []
+  private isRecording: boolean = false
+  private recordingStream: MediaStream | null = null
+  private recordingIndicator: THREE.Sprite | null = null
+  private scene: THREE.Scene
+  private renderer: THREE.WebGLRenderer
+  private eventManager: EventManagerInterface
+  private recordingStartTime: number = 0
+  private recordingDuration: number = 0
+  private recordingCanvas: HTMLCanvasElement | null = null
+
+  constructor(
+    scene: THREE.Scene,
+    renderer: THREE.WebGLRenderer,
+    eventManager: EventManagerInterface
+  ) {
+    this.scene = scene
+    this.renderer = renderer
+    this.eventManager = eventManager
+    this.setupRecordingIndicator()
+  }
+
+  private setupRecordingIndicator(): void {
+    const map = new THREE.TextureLoader().load(
+      'data:image/svg+xml;base64,' +
+        btoa(`<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64" viewBox="0 0 64 64">
+        <circle cx="32" cy="32" r="24" fill="#4CAF50" opacity="0.8" />
+        <circle cx="32" cy="32" r="16" fill="#2E7D32" opacity="0.8" />
+      </svg>`)
+    )
+    const material = new THREE.SpriteMaterial({
+      map: map,
+      transparent: true,
+      depthTest: false,
+      depthWrite: false
+    })
+    this.recordingIndicator = new THREE.Sprite(material)
+    this.recordingIndicator.scale.set(0.5, 0.5, 0.5)
+    this.recordingIndicator.position.set(-0.8, 0.8, 0)
+    this.recordingIndicator.visible = false
+
+    this.scene.add(this.recordingIndicator)
+  }
+
+  public async startRecording(): Promise<void> {
+    if (this.isRecording) {
+      return
+    }
+
+    try {
+      this.recordingCanvas = this.renderer.domElement
+
+      this.recordingStream = this.recordingCanvas.captureStream(30)
+
+      if (!this.recordingStream) {
+        throw new Error('Failed to capture stream from canvas')
+      }
+
+      this.mediaRecorder = new MediaRecorder(this.recordingStream, {
+        mimeType: 'video/webm;codecs=vp9',
+        videoBitsPerSecond: 5000000
+      })
+
+      this.recordedChunks = []
+
+      this.mediaRecorder.ondataavailable = (event) => {
+        if (event.data.size > 0) {
+          this.recordedChunks.push(event.data)
+        }
+      }
+
+      this.mediaRecorder.onstop = () => {
+        this.recordingIndicator!.visible = false
+        this.isRecording = false
+        this.recordingStream = null
+
+        this.eventManager.emitEvent('recordingStopped', {
+          duration: this.recordingDuration,
+          hasRecording: this.recordedChunks.length > 0
+        })
+      }
+
+      if (this.recordingIndicator) {
+        this.recordingIndicator.visible = true
+      }
+
+      this.mediaRecorder.start(100)
+      this.isRecording = true
+      this.recordingStartTime = Date.now()
+
+      this.eventManager.emitEvent('recordingStarted', null)
+    } catch (error) {
+      console.error('Error starting recording:', error)
+      this.eventManager.emitEvent('recordingError', error)
+    }
+  }
+
+  public stopRecording(): void {
+    if (!this.isRecording || !this.mediaRecorder) {
+      return
+    }
+
+    this.recordingDuration = (Date.now() - this.recordingStartTime) / 1000 // In seconds
+
+    this.mediaRecorder.stop()
+    if (this.recordingStream) {
+      this.recordingStream.getTracks().forEach((track) => track.stop())
+    }
+  }
+
+  public hasRecording(): boolean {
+    return this.recordedChunks.length > 0
+  }
+
+  public getRecordingDuration(): number {
+    return this.recordingDuration
+  }
+
+  public getRecordingData(): string | null {
+    if (this.recordedChunks.length !== 0) {
+      const blob = new Blob(this.recordedChunks, { type: 'video/webm' })
+
+      return URL.createObjectURL(blob)
+    }
+
+    return null
+  }
+
+  public exportRecording(filename: string = 'scene-recording.mp4'): void {
+    if (this.recordedChunks.length === 0) {
+      this.eventManager.emitEvent(
+        'recordingError',
+        new Error('No recording available to export')
+      )
+      return
+    }
+
+    this.eventManager.emitEvent('exportingRecording', null)
+
+    try {
+      const blob = new Blob(this.recordedChunks, { type: 'video/webm' })
+
+      const url = URL.createObjectURL(blob)
+      const a = document.createElement('a')
+      document.body.appendChild(a)
+      a.style.display = 'none'
+      a.href = url
+      a.download = filename
+      a.click()
+
+      window.URL.revokeObjectURL(url)
+      document.body.removeChild(a)
+
+      this.eventManager.emitEvent('recordingExported', null)
+    } catch (error) {
+      console.error('Error exporting recording:', error)
+      this.eventManager.emitEvent('recordingError', error)
+    }
+  }
+
+  public clearRecording(): void {
+    this.recordedChunks = []
+    this.recordingDuration = 0
+    this.eventManager.emitEvent('recordingCleared', null)
+  }
+
+  public dispose(): void {
+    this.stopRecording()
+    this.clearRecording()
+
+    if (this.recordingIndicator) {
+      this.scene.remove(this.recordingIndicator)
+      ;(this.recordingIndicator.material as THREE.SpriteMaterial).map?.dispose()
+      ;(this.recordingIndicator.material as THREE.SpriteMaterial).dispose()
+    }
+  }
+}

src/extensions/core/load3d/ViewHelperManager.ts

@@ -89,6 +89,16 @@ export class ViewHelperManager implements ViewHelperManagerInterface {
 
   handleResize(): void {}
 
+  visibleViewHelper(visible: boolean) {
+    if (visible) {
+      this.viewHelper.visible = true
+      this.viewHelperContainer.style.display = 'block'
+    } else {
+      this.viewHelper.visible = false
+      this.viewHelperContainer.style.display = 'none'
+    }
+  }
+
   recreateViewHelper(): void {
     if (this.viewHelper) {
       this.viewHelper.dispose()

src/extensions/core/load3d/interfaces.ts

@@ -177,3 +177,12 @@ export interface LoaderManagerInterface {
   dispose(): void
   loadModel(url: string, originalFileName?: string): Promise<void>
 }
+
+export interface RecordingManagerInterface extends BaseManager {
+  startRecording(): Promise<void>
+  stopRecording(): void
+  hasRecording(): boolean
+  getRecordingDuration(): number
+  exportRecording(filename?: string): void
+  clearRecording(): void
+}