From 31e405abc8dd1adcc8246a5eedf594ca78abae79 Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Sat, 1 Nov 2025 22:50:29 -0700 Subject: [PATCH] Feat: Loading state while loading dropped workflows (#6464) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Indicate to the user that we're hard at work parsing their JSON behind the scenes. ## Changes - **What**: Turn on the loading spinner while processing a workflow - **What else**: Refactored the code around figuring out how to grab the data from the file to make this easier ┆Issue is synchronized with this [Notion page](https://www.notion.so/PR-6464-WIP-Loading-state-for-dropped-workflows-29c6d73d3650812dba66f2a7d27a777c) by [Unito](https://www.unito.io) --------- Co-authored-by: GitHub Action --- src/scripts/app.ts | 331 ++++++------------------- src/scripts/metadata/json.ts | 29 +++ src/scripts/metadata/parser.ts | 75 ++++++ src/scripts/metadata/png.ts | 4 +- src/scripts/pnginfo.ts | 8 +- src/scripts/ui.ts | 8 +- src/utils/__tests__/eventUtils.test.ts | 68 +++++ src/utils/eventUtils.ts | 27 ++ 8 files changed, 287 insertions(+), 263 deletions(-) create mode 100644 src/scripts/metadata/json.ts create mode 100644 src/scripts/metadata/parser.ts create mode 100644 src/utils/__tests__/eventUtils.test.ts create mode 100644 src/utils/eventUtils.ts diff --git a/src/scripts/app.ts b/src/scripts/app.ts index 6fa25d051..a0479b5b2 100644 --- a/src/scripts/app.ts +++ b/src/scripts/app.ts @@ -1,4 +1,4 @@ -import { useResizeObserver } from '@vueuse/core' +import { useEventListener, useResizeObserver } from '@vueuse/core' import _ from 'es-toolkit/compat' import type { ToastMessageOptions } from 'primevue/toast' import { reactive, unref } from 'vue' @@ -42,12 +42,6 @@ import { isComboInputSpecV2 } from '@/schemas/nodeDefSchema' import { type BaseDOMWidget, DOMWidgetImpl } from '@/scripts/domWidget' -import { getFromWebmFile } from '@/scripts/metadata/ebml' -import { getGltfBinaryMetadata } from '@/scripts/metadata/gltf' -import { getFromIsobmffFile } from '@/scripts/metadata/isobmff' -import { getMp3Metadata } from '@/scripts/metadata/mp3' -import { getOggMetadata } from '@/scripts/metadata/ogg' -import { getSvgMetadata } from '@/scripts/metadata/svg' import { useDialogService } from '@/services/dialogService' import { useSubscription } from '@/platform/cloud/subscription/composables/useSubscription' import { useExtensionService } from '@/services/extensionService' @@ -89,19 +83,14 @@ import { deserialiseAndCreate } from '@/utils/vintageClipboard' import { type ComfyApi, PromptExecutionError, api } from './api' import { defaultGraph } from './defaultGraph' -import { - getAvifMetadata, - getFlacMetadata, - getLatentMetadata, - getPngMetadata, - getWebpMetadata, - importA1111 -} from './pnginfo' +import { importA1111 } from './pnginfo' import { $el, ComfyUI } from './ui' import { ComfyAppMenu } from './ui/menu/index' import { clone } from './utils' import { type ComfyWidgetConstructor } from './widgets' import { ensureCorrectLayoutScale } from '@/renderer/extensions/vueNodes/layout/ensureCorrectLayoutScale' +import { extractFileFromDragEvent } from '@/utils/eventUtils' +import { getWorkflowDataFromFile } from '@/scripts/metadata/parser' export const ANIM_PREVIEW_WIDGET = '$$comfy_animation_preview' @@ -534,7 +523,7 @@ export class ComfyApp { */ private addDropHandler() { // Get prompt from dropped PNG or json - document.addEventListener('drop', async (event) => { + useEventListener(document, 'drop', async (event: DragEvent) => { try { event.preventDefault() event.stopPropagation() @@ -543,66 +532,49 @@ export class ComfyApp { this.dragOverNode = null // Node handles file drop, we dont use the built in onDropFile handler as its buggy // If you drag multiple files it will call it multiple times with the same file - if (n && n.onDragDrop && (await n.onDragDrop(event))) { - return + if (await n?.onDragDrop?.(event)) return + + const fileMaybe = await extractFileFromDragEvent(event) + if (!fileMaybe) return + + const workspace = useWorkspaceStore() + try { + workspace.spinner = true + await this.handleFile(fileMaybe, 'file_drop') + } finally { + workspace.spinner = false } - // Dragging from Chrome->Firefox there is a file but its a bmp, so ignore that - if (!event.dataTransfer) return - if ( - event.dataTransfer.files.length && - event.dataTransfer.files[0].type !== 'image/bmp' - ) { - await this.handleFile(event.dataTransfer.files[0], 'file_drop') - } else { - // Try loading the first URI in the transfer list - const validTypes = ['text/uri-list', 'text/x-moz-url'] - const match = [...event.dataTransfer.types].find((t) => - validTypes.find((v) => t === v) - ) - if (match) { - const uri = event.dataTransfer.getData(match)?.split('\n')?.[0] - if (uri) { - const blob = await (await fetch(uri)).blob() - await this.handleFile( - new File([blob], uri, { type: blob.type }), - 'file_drop' - ) - } - } - } - } catch (err: any) { - useToastStore().addAlert( - t('toastMessages.dropFileError', { error: err }) - ) + } catch (error: unknown) { + useToastStore().addAlert(t('toastMessages.dropFileError', { error })) } }) // Always clear over node on drag leave - this.canvasEl.addEventListener('dragleave', async () => { - if (this.dragOverNode) { - this.dragOverNode = null - this.graph.setDirtyCanvas(false, true) - } + useEventListener(this.canvasElRef, 'dragleave', async () => { + if (!this.dragOverNode) return + this.dragOverNode = null + this.graph.setDirtyCanvas(false, true) }) // Add handler for dropping onto a specific node - this.canvasEl.addEventListener( + useEventListener( + this.canvasElRef, 'dragover', - (e) => { - this.canvas.adjustMouseEvent(e) - const node = this.graph.getNodeOnPos(e.canvasX, e.canvasY) - if (node) { - if (node.onDragOver && node.onDragOver(e)) { - this.dragOverNode = node + (event: DragEvent) => { + this.canvas.adjustMouseEvent(event) + const node = this.graph.getNodeOnPos(event.canvasX, event.canvasY) - // dragover event is fired very frequently, run this on an animation frame - requestAnimationFrame(() => { - this.graph.setDirtyCanvas(false, true) - }) - return - } + if (!node?.onDragOver?.(event)) { + this.dragOverNode = null + return } - this.dragOverNode = null + + this.dragOverNode = node + + // dragover event is fired very frequently, run this on an animation frame + requestAnimationFrame(() => { + this.graph.setDirtyCanvas(false, true) + }) }, false ) @@ -1417,199 +1389,50 @@ export class ComfyApp { * @param {File} file */ async handleFile(file: File, openSource?: WorkflowOpenSource) { - const removeExt = (f: string) => { - if (!f) return f - const p = f.lastIndexOf('.') - if (p === -1) return f - return f.substring(0, p) - } - const fileName = removeExt(file.name) - if (file.type === 'image/png') { - const pngInfo = await getPngMetadata(file) - if (pngInfo?.workflow) { - await this.loadGraphData( - JSON.parse(pngInfo.workflow), - true, - true, - fileName, - { openSource } - ) - } else if (pngInfo?.prompt) { - this.loadApiJson(JSON.parse(pngInfo.prompt), fileName) - } else if (pngInfo?.parameters) { - // Note: Not putting this in `importA1111` as it is mostly not used - // by external callers, and `importA1111` has no access to `app`. - useWorkflowService().beforeLoadNewGraph() - importA1111(this.graph, pngInfo.parameters) - useWorkflowService().afterLoadNewGraph( - fileName, - this.graph.serialize() as unknown as ComfyWorkflowJSON - ) - } else { - this.showErrorOnFileLoad(file) - } - } else if (file.type === 'image/avif') { - const { workflow, prompt } = await getAvifMetadata(file) - - if (workflow) { - this.loadGraphData(JSON.parse(workflow), true, true, fileName, { - openSource - }) - } else if (prompt) { - this.loadApiJson(JSON.parse(prompt), fileName) - } else { - this.showErrorOnFileLoad(file) - } - } else if (file.type === 'image/webp') { - const pngInfo = await getWebpMetadata(file) - // Support loading workflows from that webp custom node. - const workflow = pngInfo?.workflow || pngInfo?.Workflow - const prompt = pngInfo?.prompt || pngInfo?.Prompt - - if (workflow) { - this.loadGraphData(JSON.parse(workflow), true, true, fileName, { - openSource - }) - } else if (prompt) { - this.loadApiJson(JSON.parse(prompt), fileName) - } else { - this.showErrorOnFileLoad(file) - } - } else if (file.type === 'audio/mpeg') { - const { workflow, prompt } = await getMp3Metadata(file) - if (workflow) { - this.loadGraphData(workflow, true, true, fileName, { openSource }) - } else if (prompt) { - this.loadApiJson(prompt, fileName) - } else { - this.showErrorOnFileLoad(file) - } - } else if (file.type === 'audio/ogg') { - const { workflow, prompt } = await getOggMetadata(file) - if (workflow) { - this.loadGraphData(workflow, true, true, fileName, { openSource }) - } else if (prompt) { - this.loadApiJson(prompt, fileName) - } else { - this.showErrorOnFileLoad(file) - } - } else if (file.type === 'audio/flac' || file.type === 'audio/x-flac') { - const pngInfo = await getFlacMetadata(file) - const workflow = pngInfo?.workflow || pngInfo?.Workflow - const prompt = pngInfo?.prompt || pngInfo?.Prompt - - if (workflow) { - this.loadGraphData(JSON.parse(workflow), true, true, fileName, { - openSource - }) - } else if (prompt) { - this.loadApiJson(JSON.parse(prompt), fileName) - } else { - this.showErrorOnFileLoad(file) - } - } else if (file.type === 'video/webm') { - const webmInfo = await getFromWebmFile(file) - if (webmInfo.workflow) { - this.loadGraphData(webmInfo.workflow, true, true, fileName, { - openSource - }) - } else if (webmInfo.prompt) { - this.loadApiJson(webmInfo.prompt, fileName) - } else { - this.showErrorOnFileLoad(file) - } - } else if ( - file.type === 'video/mp4' || - file.name?.endsWith('.mp4') || - file.name?.endsWith('.mov') || - file.name?.endsWith('.m4v') || - file.type === 'video/quicktime' || - file.type === 'video/x-m4v' - ) { - const mp4Info = await getFromIsobmffFile(file) - if (mp4Info.workflow) { - this.loadGraphData(mp4Info.workflow, true, true, fileName, { - openSource - }) - } else if (mp4Info.prompt) { - this.loadApiJson(mp4Info.prompt, fileName) - } - } else if (file.type === 'image/svg+xml' || file.name?.endsWith('.svg')) { - const svgInfo = await getSvgMetadata(file) - if (svgInfo.workflow) { - this.loadGraphData(svgInfo.workflow, true, true, fileName, { - openSource - }) - } else if (svgInfo.prompt) { - this.loadApiJson(svgInfo.prompt, fileName) - } else { - this.showErrorOnFileLoad(file) - } - } else if ( - file.type === 'model/gltf-binary' || - file.name?.endsWith('.glb') - ) { - const gltfInfo = await getGltfBinaryMetadata(file) - if (gltfInfo.workflow) { - this.loadGraphData(gltfInfo.workflow, true, true, fileName, { - openSource - }) - } else if (gltfInfo.prompt) { - this.loadApiJson(gltfInfo.prompt, fileName) - } else { - this.showErrorOnFileLoad(file) - } - } else if ( - file.type === 'application/json' || - file.name?.endsWith('.json') - ) { - const reader = new FileReader() - reader.onload = async () => { - const readerResult = reader.result as string - const jsonContent = JSON.parse(readerResult) - if (jsonContent?.templates) { - this.loadTemplateData(jsonContent) - } else if (this.isApiJson(jsonContent)) { - this.loadApiJson(jsonContent, fileName) - } else { - await this.loadGraphData( - JSON.parse(readerResult), - true, - true, - fileName, - { openSource } - ) - } - } - reader.readAsText(file) - } else if ( - file.name?.endsWith('.latent') || - file.name?.endsWith('.safetensors') - ) { - const info = await getLatentMetadata(file) - // TODO define schema to LatentMetadata - // @ts-expect-error - if (info.workflow) { - await this.loadGraphData( - // @ts-expect-error - JSON.parse(info.workflow), - true, - true, - fileName, - { openSource } - ) - // @ts-expect-error - } else if (info.prompt) { - // @ts-expect-error - this.loadApiJson(JSON.parse(info.prompt)) - } else { - this.showErrorOnFileLoad(file) - } - } else { + const fileName = file.name.replace(/\.\w+$/, '') // Strip file extension + const workflowData = await getWorkflowDataFromFile(file) + if (!workflowData) { this.showErrorOnFileLoad(file) + return } + + const { workflow, prompt, parameters, templates } = workflowData + + if (templates) { + this.loadTemplateData({ templates }) + } + + if (parameters) { + // Note: Not putting this in `importA1111` as it is mostly not used + // by external callers, and `importA1111` has no access to `app`. + useWorkflowService().beforeLoadNewGraph() + importA1111(this.graph, parameters) + useWorkflowService().afterLoadNewGraph( + fileName, + this.graph.serialize() as unknown as ComfyWorkflowJSON + ) + return + } + + if (workflow) { + const workflowObj = + typeof workflow === 'string' ? JSON.parse(workflow) : workflow + await this.loadGraphData(workflowObj, true, true, fileName, { + openSource + }) + return + } + + if (prompt) { + const promptObj = typeof prompt === 'string' ? JSON.parse(prompt) : prompt + this.loadApiJson(promptObj, fileName) + return + } + + this.showErrorOnFileLoad(file) } + // @deprecated isApiJson(data: unknown) { return _.isObject(data) && Object.values(data).every((v) => v.class_type) } diff --git a/src/scripts/metadata/json.ts b/src/scripts/metadata/json.ts new file mode 100644 index 000000000..873b42117 --- /dev/null +++ b/src/scripts/metadata/json.ts @@ -0,0 +1,29 @@ +import { isObject } from 'es-toolkit/compat' + +export function getDataFromJSON( + file: File +): Promise | undefined> { + return new Promise | undefined>((resolve) => { + const reader = new FileReader() + reader.onload = async () => { + const readerResult = reader.result as string + const jsonContent = JSON.parse(readerResult) + if (jsonContent?.templates) { + resolve({ templates: jsonContent.templates }) + return + } + if (isApiJson(jsonContent)) { + resolve({ prompt: jsonContent }) + return + } + resolve({ workflow: jsonContent }) + return + } + reader.readAsText(file) + return + }) +} + +function isApiJson(data: unknown) { + return isObject(data) && Object.values(data).every((v) => v.class_type) +} diff --git a/src/scripts/metadata/parser.ts b/src/scripts/metadata/parser.ts new file mode 100644 index 000000000..f11b9bef7 --- /dev/null +++ b/src/scripts/metadata/parser.ts @@ -0,0 +1,75 @@ +import { getFromWebmFile } from '@/scripts/metadata/ebml' +import { getGltfBinaryMetadata } from '@/scripts/metadata/gltf' +import { getFromIsobmffFile } from '@/scripts/metadata/isobmff' +import { getDataFromJSON } from '@/scripts/metadata/json' +import { getMp3Metadata } from '@/scripts/metadata/mp3' +import { getOggMetadata } from '@/scripts/metadata/ogg' +import { getSvgMetadata } from '@/scripts/metadata/svg' +import { + getAvifMetadata, + getWebpMetadata, + getFlacMetadata, + getLatentMetadata, + getPngMetadata +} from '@/scripts/pnginfo' + +export async function getWorkflowDataFromFile( + file: File +): Promise | undefined> { + if (file.type === 'image/png') { + return await getPngMetadata(file) + } + if (file.type === 'image/avif') { + return await getAvifMetadata(file) + } + if (file.type === 'image/webp') { + const pngInfo = await getWebpMetadata(file) + // Support loading workflows from that webp custom node. + const workflow = pngInfo?.workflow || pngInfo?.Workflow + const prompt = pngInfo?.prompt || pngInfo?.Prompt + return { workflow, prompt } + } + if (file.type === 'audio/mpeg') { + return await getMp3Metadata(file) + } + if (file.type === 'audio/ogg') { + return await getOggMetadata(file) + } + if (file.type === 'audio/flac' || file.type === 'audio/x-flac') { + const pngInfo = await getFlacMetadata(file) + const workflow = pngInfo?.workflow || pngInfo?.Workflow + const prompt = pngInfo?.prompt || pngInfo?.Prompt + + return { workflow, prompt } + } + if (file.type === 'video/webm') { + return (await getFromWebmFile(file)) as unknown as Record + } + if ( + file.name?.endsWith('.mp4') || + file.name?.endsWith('.mov') || + file.name?.endsWith('.m4v') || + file.type === 'video/mp4' || + file.type === 'video/quicktime' || + file.type === 'video/x-m4v' + ) { + return (await getFromIsobmffFile(file)) as unknown as Record + } + if (file.type === 'image/svg+xml' || file.name?.endsWith('.svg')) { + return (await getSvgMetadata(file)) as unknown as Record + } + if (file.type === 'model/gltf-binary' || file.name?.endsWith('.glb')) { + return (await getGltfBinaryMetadata(file)) as unknown as Record< + string, + object + > + } + if (file.name?.endsWith('.latent') || file.name?.endsWith('.safetensors')) { + return await getLatentMetadata(file) + } + + if (file.type === 'application/json' || file.name?.endsWith('.json')) { + return getDataFromJSON(file) + } + return +} diff --git a/src/scripts/metadata/png.ts b/src/scripts/metadata/png.ts index af3be9a64..2e911796a 100644 --- a/src/scripts/metadata/png.ts +++ b/src/scripts/metadata/png.ts @@ -1,5 +1,5 @@ /** @knipIgnoreUnusedButUsedByCustomNodes */ -export function getFromPngBuffer(buffer: ArrayBuffer) { +export function getFromPngBuffer(buffer: ArrayBuffer): Record { // Get the PNG data as a Uint8Array const pngData = new Uint8Array(buffer) const dataView = new DataView(pngData.buffer) @@ -7,7 +7,7 @@ export function getFromPngBuffer(buffer: ArrayBuffer) { // Check that the PNG signature is present if (dataView.getUint32(0) !== 0x89504e47) { console.error('Not a valid PNG file') - return + return {} } // Start searching for chunks after the PNG signature diff --git a/src/scripts/pnginfo.ts b/src/scripts/pnginfo.ts index 7d6eea535..97c6a9320 100644 --- a/src/scripts/pnginfo.ts +++ b/src/scripts/pnginfo.ts @@ -138,13 +138,13 @@ export function getWebpMetadata(file) { }) } -// @ts-expect-error fixme ts strict error -export function getLatentMetadata(file) { +export function getLatentMetadata(file: File): Promise> { return new Promise((r) => { const reader = new FileReader() reader.onload = (event) => { - // @ts-expect-error fixme ts strict error - const safetensorsData = new Uint8Array(event.target.result as ArrayBuffer) + const safetensorsData = new Uint8Array( + event.target?.result as ArrayBuffer + ) const dataView = new DataView(safetensorsData.buffer) let header_size = dataView.getUint32(0, true) let offset = 8 diff --git a/src/scripts/ui.ts b/src/scripts/ui.ts index b8c92b71f..67d190f1f 100644 --- a/src/scripts/ui.ts +++ b/src/scripts/ui.ts @@ -393,9 +393,11 @@ export class ComfyUI { style: { display: 'none' }, parent: document.body, onchange: async () => { - // @ts-expect-error fixme ts strict error - await app.handleFile(fileInput.files[0], 'file_button') - fileInput.value = '' + const file = fileInput.files?.[0] + if (file) { + await app.handleFile(file, 'file_button') + fileInput.value = '' + } } }) diff --git a/src/utils/__tests__/eventUtils.test.ts b/src/utils/__tests__/eventUtils.test.ts new file mode 100644 index 000000000..2fc51ac67 --- /dev/null +++ b/src/utils/__tests__/eventUtils.test.ts @@ -0,0 +1,68 @@ +import { extractFileFromDragEvent } from '@/utils/eventUtils' +import { describe, expect, it } from 'vitest' + +describe('eventUtils', () => { + describe('extractFileFromDragEvent', () => { + it('should handle drops with no data', async () => { + const actual = await extractFileFromDragEvent(new FakeDragEvent('drop')) + expect(actual).toBe(undefined) + }) + + it('should handle drops with dataTransfer but no files', async () => { + const actual = await extractFileFromDragEvent( + new FakeDragEvent('drop', { dataTransfer: new DataTransfer() }) + ) + expect(actual).toBe(undefined) + }) + + it('should handle drops with dataTransfer with files', async () => { + const fileWithWorkflowMaybeWhoKnows = new File( + [new Uint8Array()], + 'fake_workflow.json', + { + type: 'application/json' + } + ) + + const dataTransfer = new DataTransfer() + dataTransfer.items.add(fileWithWorkflowMaybeWhoKnows) + + const event = new FakeDragEvent('drop', { dataTransfer }) + + const actual = await extractFileFromDragEvent(event) + expect(actual).toBe(fileWithWorkflowMaybeWhoKnows) + }) + + // Skip until we can setup MSW + it.skip('should handle drops with URLs', async () => { + const urlWithWorkflow = 'https://fakewebsite.notreal/fake_workflow.json' + + const dataTransfer = new DataTransfer() + + dataTransfer.setData('text/uri-list', urlWithWorkflow) + dataTransfer.setData('text/x-moz-url', urlWithWorkflow) + + const event = new FakeDragEvent('drop', { dataTransfer }) + + const actual = await extractFileFromDragEvent(event) + expect(actual).toBeInstanceOf(File) + }) + }) +}) + +// Needed to keep the dataTransfer defined +class FakeDragEvent extends DragEvent { + override dataTransfer: DataTransfer | null + override clientX: number + override clientY: number + + constructor( + type: string, + { dataTransfer, clientX, clientY }: DragEventInit = {} + ) { + super(type) + this.dataTransfer = dataTransfer ?? null + this.clientX = clientX ?? 0 + this.clientY = clientY ?? 0 + } +} diff --git a/src/utils/eventUtils.ts b/src/utils/eventUtils.ts new file mode 100644 index 000000000..133ccd709 --- /dev/null +++ b/src/utils/eventUtils.ts @@ -0,0 +1,27 @@ +export async function extractFileFromDragEvent( + event: DragEvent +): Promise { + if (!event.dataTransfer) return + + // Dragging from Chrome->Firefox there is a file but its a bmp, so ignore that + if ( + event.dataTransfer.files.length && + event.dataTransfer.files[0].type !== 'image/bmp' + ) { + return event.dataTransfer.files[0] + } + + // Try loading the first URI in the transfer list + const validTypes = ['text/uri-list', 'text/x-moz-url'] + const match = [...event.dataTransfer.types].find((t) => + validTypes.includes(t) + ) + if (!match) return + + const uri = event.dataTransfer.getData(match)?.split('\n')?.[0] + if (!uri) return + + const response = await fetch(uri) + const blob = await response.blob() + return new File([blob], uri, { type: blob.type }) +}