From f4b5677901ea4d36493479faa6279f382158a0a0 Mon Sep 17 00:00:00 2001 From: Terry Jia Date: Thu, 2 Jan 2025 15:45:37 -0500 Subject: [PATCH] [3d] support for fov and mask (#2116) --- src/extensions/core/load3d.ts | 137 +++++++++++++++++++++------------- 1 file changed, 87 insertions(+), 50 deletions(-) diff --git a/src/extensions/core/load3d.ts b/src/extensions/core/load3d.ts index 5900be2580..5a00648e55 100644 --- a/src/extensions/core/load3d.ts +++ b/src/extensions/core/load3d.ts @@ -13,6 +13,30 @@ import { api } from '@/scripts/api' import { app } from '@/scripts/app' import { useToastStore } from '@/stores/toastStore' +async function uploadTempImage(imageData, prefix) { + const blob = await fetch(imageData).then((r) => r.blob()) + const name = `${prefix}_${Date.now()}.png` + const file = new File([blob], name) + + const body = new FormData() + body.append('image', file) + body.append('subfolder', 'threed') + body.append('type', 'temp') + + const resp = await api.fetchApi('/upload/image', { + method: 'POST', + body + }) + + if (resp.status !== 200) { + const err = `Error uploading temp image: ${resp.status} - ${resp.statusText}` + useToastStore().addAlert(err) + throw new Error(err) + } + + return await resp.json() +} + async function uploadFile( load3d: Load3d, file: File, @@ -130,7 +154,7 @@ class Load3d { this.perspectiveCamera.lookAt(0, 0, 0) this.orthographicCamera.lookAt(0, 0, 0) - this.renderer = new THREE.WebGLRenderer({ antialias: true }) + this.renderer = new THREE.WebGLRenderer({ alpha: true, antialias: true }) this.renderer.setSize(300, 300) this.renderer.setClearColor(0x282828) @@ -186,6 +210,14 @@ class Load3d { this.startAnimation() } + setFOV(fov: number) { + if (this.activeCamera === this.perspectiveCamera) { + this.perspectiveCamera.fov = fov + this.perspectiveCamera.updateProjectionMatrix() + this.renderer.render(this.scene, this.activeCamera) + } + } + getCameraState() { const currentType = this.getCurrentCameraType() return { @@ -758,11 +790,18 @@ class Load3d { this.renderer.render(this.scene, this.activeCamera) } - captureScene(width: number, height: number): Promise { - return new Promise((resolve, reject) => { + captureScene( + width: number, + height: number + ): Promise<{ scene: string; mask: string }> { + return new Promise(async (resolve, reject) => { try { const originalWidth = this.renderer.domElement.width const originalHeight = this.renderer.domElement.height + const originalClearColor = this.renderer.getClearColor( + new THREE.Color() + ) + const originalClearAlpha = this.renderer.getClearAlpha() this.renderer.setSize(width, height) @@ -780,13 +819,17 @@ class Load3d { } this.renderer.render(this.scene, this.activeCamera) + const sceneData = this.renderer.domElement.toDataURL('image/png') - const imageData = this.renderer.domElement.toDataURL('image/png') + this.renderer.setClearColor(0x000000, 0) + this.renderer.render(this.scene, this.activeCamera) + const maskData = this.renderer.domElement.toDataURL('image/png') + this.renderer.setClearColor(originalClearColor, originalClearAlpha) this.renderer.setSize(originalWidth, originalHeight) this.handleResize() - resolve(imageData) + resolve({ scene: sceneData, mask: maskData }) } catch (error) { reject(error) } @@ -1040,6 +1083,7 @@ function configureLoad3D( bgColor: IWidget, lightIntensity: IWidget, upDirection: IWidget, + fov: IWidget, cameraState?: any, postModelUpdateFunc?: (load3d: Load3d) => void ) { @@ -1137,6 +1181,12 @@ function configureLoad3D( load3d.setUpDirection( upDirection.value as 'original' | '-x' | '+x' | '-y' | '+y' | '-z' | '+z' ) + + fov.callback = (value: number) => { + load3d.setFOV(value) + } + + load3d.setFOV(fov.value as number) } app.registerExtension({ @@ -1282,6 +1332,8 @@ app.registerExtension({ (w: IWidget) => w.name === 'up_direction' ) + const fov = node.widgets.find((w: IWidget) => w.name === 'fov') + let cameraState try { const cameraInfo = node.properties['Camera Info'] @@ -1308,6 +1360,7 @@ app.registerExtension({ bgColor, lightIntensity, upDirection, + fov, cameraState ) @@ -1318,30 +1371,20 @@ app.registerExtension({ sceneWidget.serializeValue = async () => { node.properties['Camera Info'] = JSON.stringify(load3d.getCameraState()) - const imageData = await load3d.captureScene(w.value, h.value) + const { scene: imageData, mask: maskData } = await load3d.captureScene( + w.value, + h.value + ) - const blob = await fetch(imageData).then((r) => r.blob()) - const name = `scene_${Date.now()}.png` - const file = new File([blob], name) + const [data, dataMask] = await Promise.all([ + uploadTempImage(imageData, 'scene'), + uploadTempImage(maskData, 'scene_mask') + ]) - const body = new FormData() - body.append('image', file) - body.append('subfolder', 'threed') - body.append('type', 'temp') - - const resp = await api.fetchApi('/upload/image', { - method: 'POST', - body - }) - - if (resp.status !== 200) { - const err = `Error uploading scene capture: ${resp.status} - ${resp.statusText}` - useToastStore().addAlert(err) - throw new Error(err) + return { + image: `threed/${data.name} [temp]`, + mask: `threed/${dataMask.name} [temp]` } - - const data = await resp.json() - return `threed/${data.name} [temp]` } } }) @@ -1557,6 +1600,8 @@ app.registerExtension({ } } + const fov = node.widgets.find((w: IWidget) => w.name === 'fov') + let cameraState try { const cameraInfo = node.properties['Camera Info'] @@ -1583,6 +1628,7 @@ app.registerExtension({ bgColor, lightIntensity, upDirection, + fov, cameraState, (load3d: Load3d) => { const animationLoad3d = load3d as Load3dAnimation @@ -1606,32 +1652,20 @@ app.registerExtension({ sceneWidget.serializeValue = async () => { node.properties['Camera Info'] = JSON.stringify(load3d.getCameraState()) - load3d.toggleAnimation(false) + const { scene: imageData, mask: maskData } = await load3d.captureScene( + w.value, + h.value + ) - const imageData = await load3d.captureScene(w.value, h.value) + const [data, dataMask] = await Promise.all([ + uploadTempImage(imageData, 'scene'), + uploadTempImage(maskData, 'scene_mask') + ]) - const blob = await fetch(imageData).then((r) => r.blob()) - const name = `scene_${Date.now()}.png` - const file = new File([blob], name) - - const body = new FormData() - body.append('image', file) - body.append('subfolder', 'threed') - body.append('type', 'temp') - - const resp = await api.fetchApi('/upload/image', { - method: 'POST', - body - }) - - if (resp.status !== 200) { - const err = `Error uploading scene capture: ${resp.status} - ${resp.statusText}` - useToastStore().addAlert(err) - throw new Error(err) + return { + image: `threed/${data.name} [temp]`, + mask: `threed/${dataMask.name} [temp]` } - - const data = await resp.json() - return `threed/${data.name} [temp]` } } }) @@ -1744,6 +1778,8 @@ app.registerExtension({ (w: IWidget) => w.name === 'up_direction' ) + const fov = node.widgets.find((w: IWidget) => w.name === 'fov') + const onExecuted = node.onExecuted node.onExecuted = function (message: any) { @@ -1771,7 +1807,8 @@ app.registerExtension({ material, bgColor, lightIntensity, - upDirection + upDirection, + fov ) } }