diff --git a/browser_tests/assets/workflow.m4v b/browser_tests/assets/workflow.m4v new file mode 100644 index 000000000..3e1168905 Binary files /dev/null and b/browser_tests/assets/workflow.m4v differ diff --git a/browser_tests/assets/workflow.mov b/browser_tests/assets/workflow.mov new file mode 100644 index 000000000..c3c30357f Binary files /dev/null and b/browser_tests/assets/workflow.mov differ diff --git a/browser_tests/assets/workflow.mp4 b/browser_tests/assets/workflow.mp4 new file mode 100644 index 000000000..830e92314 Binary files /dev/null and b/browser_tests/assets/workflow.mp4 differ diff --git a/browser_tests/tests/loadWorkflowInMedia.spec.ts b/browser_tests/tests/loadWorkflowInMedia.spec.ts index 561ed7491..39df2d77b 100644 --- a/browser_tests/tests/loadWorkflowInMedia.spec.ts +++ b/browser_tests/tests/loadWorkflowInMedia.spec.ts @@ -9,7 +9,10 @@ test.describe('Load Workflow in Media', () => { 'no_workflow.webp', 'large_workflow.webp', 'workflow.webm', - 'workflow.glb' + 'workflow.glb', + 'workflow.mp4', + 'workflow.mov', + 'workflow.m4v' ] fileNames.forEach(async (fileName) => { test(`Load workflow in ${fileName} (drop from filesystem)`, async ({ diff --git a/browser_tests/tests/loadWorkflowInMedia.spec.ts-snapshots/workflow-m4v-chromium-linux.png b/browser_tests/tests/loadWorkflowInMedia.spec.ts-snapshots/workflow-m4v-chromium-linux.png new file mode 100644 index 000000000..387644ce3 Binary files /dev/null and b/browser_tests/tests/loadWorkflowInMedia.spec.ts-snapshots/workflow-m4v-chromium-linux.png differ diff --git a/browser_tests/tests/loadWorkflowInMedia.spec.ts-snapshots/workflow-mov-chromium-linux.png b/browser_tests/tests/loadWorkflowInMedia.spec.ts-snapshots/workflow-mov-chromium-linux.png new file mode 100644 index 000000000..387644ce3 Binary files /dev/null and b/browser_tests/tests/loadWorkflowInMedia.spec.ts-snapshots/workflow-mov-chromium-linux.png differ diff --git a/browser_tests/tests/loadWorkflowInMedia.spec.ts-snapshots/workflow-mp4-chromium-linux.png b/browser_tests/tests/loadWorkflowInMedia.spec.ts-snapshots/workflow-mp4-chromium-linux.png new file mode 100644 index 000000000..387644ce3 Binary files /dev/null and b/browser_tests/tests/loadWorkflowInMedia.spec.ts-snapshots/workflow-mp4-chromium-linux.png differ diff --git a/src/scripts/app.ts b/src/scripts/app.ts index 602fbd55a..26b9e41d1 100644 --- a/src/scripts/app.ts +++ b/src/scripts/app.ts @@ -27,6 +27,7 @@ import { import type { ComfyNodeDef as ComfyNodeDefV1 } from '@/schemas/nodeDefSchema' import { getFromWebmFile } from '@/scripts/metadata/ebml' import { getGltfBinaryMetadata } from '@/scripts/metadata/gltf' +import { getFromIsobmffFile } from '@/scripts/metadata/isobmff' import { useDialogService } from '@/services/dialogService' import { useExtensionService } from '@/services/extensionService' import { useLitegraphService } from '@/services/litegraphService' @@ -1333,6 +1334,20 @@ export class ComfyApp { } else { this.showErrorOnFileLoad(file) } + } else if ( + file.type === 'video/mp4' || + file.name?.endsWith('.mp4') || + file.name?.endsWith('.mov') || + file.name?.endsWith('.m4v') || + file.type === 'video/quicktime' || + file.type === 'video/x-m4v' + ) { + const mp4Info = await getFromIsobmffFile(file) + if (mp4Info.workflow) { + this.loadGraphData(mp4Info.workflow, true, true, fileName) + } else if (mp4Info.prompt) { + this.loadApiJson(mp4Info.prompt, fileName) + } } else if ( file.type === 'model/gltf-binary' || file.name?.endsWith('.glb') diff --git a/src/scripts/metadata/isobmff.ts b/src/scripts/metadata/isobmff.ts new file mode 100644 index 000000000..cf0bab35f --- /dev/null +++ b/src/scripts/metadata/isobmff.ts @@ -0,0 +1,272 @@ +import { + ComfyApiWorkflow, + ComfyWorkflowJSON +} from '@/schemas/comfyWorkflowSchema' +import { + ASCII, + ComfyMetadata, + ComfyMetadataTags, + IsobmffBoxContentRange +} from '@/types/metadataTypes' + +const MAX_READ_BYTES = 2 * 1024 * 1024 +const BOX_TYPES = { + USER_DATA: [0x75, 0x64, 0x74, 0x61], + META_DATA: [0x6d, 0x65, 0x74, 0x61], + ITEM_LIST: [0x69, 0x6c, 0x73, 0x74], + KEYS: [0x6b, 0x65, 0x79, 0x73], + DATA: [0x64, 0x61, 0x74, 0x61], + MOVIE: [0x6d, 0x6f, 0x6f, 0x76] +} +const SIZES = { + HEADER: 8, + VERSION: 4, + LOCALE: 4, + ITEM_MIN: 8 +} + +const bufferMatchesBoxType = ( + data: Uint8Array, + pos: number, + boxType: number[] +): boolean => { + if (pos + 4 > data.length) return false + + for (let i = 0; i < 4; i++) { + if (data[pos + i] !== boxType[i]) return false + } + return true +} + +const readUint32 = (data: Uint8Array, pos: number): number => { + if (pos + 4 > data.length) return 0 + return ( + (data[pos] << 24) | + (data[pos + 1] << 16) | + (data[pos + 2] << 8) | + data[pos + 3] + ) +} + +const findIsobmffBoxByType = ( + data: Uint8Array, + startPos: number, + endPos: number, + boxType: number[] +): IsobmffBoxContentRange => { + for (let pos = startPos; pos < endPos - 8; pos++) { + const size = readUint32(data, pos) + if (size < SIZES.ITEM_MIN) continue // Minimum size is 8 bytes + + if (bufferMatchesBoxType(data, pos + 4, boxType)) + return { start: pos + SIZES.HEADER, end: pos + size } // Skip header + + // If type doesn't match, ensure size is valid before skipping + if (pos + size > endPos) return null + + pos += size - 1 // Skip to the next potential box start + } + return null +} + +const extractJson = (data: Uint8Array, start: number, end: number): any => { + let jsonStart = start + while (jsonStart < end && data[jsonStart] !== ASCII.OPEN_BRACE) { + jsonStart++ + } + if (jsonStart >= end) return null + + try { + const jsonText = new TextDecoder().decode(data.slice(jsonStart, end)) + return JSON.parse(jsonText) + } catch { + return null + } +} + +const readUtf8String = (data: Uint8Array, start: number, end: number): string => + new TextDecoder().decode(data.slice(start, end)) + +const parseKeysBox = ( + data: Uint8Array, + keysBoxStart: number, + keysBoxEnd: number +): Map => { + const keysMap = new Map() + let pos = keysBoxStart + 4 // Skip version/flags + if (pos + 4 > keysBoxEnd) return keysMap + + const entryCount = readUint32(data, pos) + pos += 4 + + for (let i = 1; i <= entryCount; i++) { + // Keys are 1-indexed + if (pos + SIZES.HEADER > keysBoxEnd) break + + const keySize = readUint32(data, pos) + pos += SIZES.HEADER + + const keyNameEnd = pos + keySize - SIZES.HEADER + if (keySize < SIZES.ITEM_MIN || keyNameEnd > keysBoxEnd) break + + const keyName = readUtf8String(data, pos, keyNameEnd) + keysMap.set(i, keyName) + pos = keyNameEnd + } + return keysMap +} + +const extractMetadataValueFromDataBox = ( + data: Uint8Array, + dataBoxStart: number, + dataBoxEnd: number, + keyName: string +): ComfyWorkflowJSON | ComfyApiWorkflow | null => { + const valueStart = dataBoxStart + SIZES.VERSION + SIZES.LOCALE + if (valueStart >= dataBoxEnd) return null + + const lowerKeyName = keyName.toLowerCase() + if ( + lowerKeyName === ComfyMetadataTags.PROMPT.toLowerCase() || + lowerKeyName === ComfyMetadataTags.WORKFLOW.toLowerCase() + ) { + return extractJson(data, valueStart, dataBoxEnd) || null + } + return null +} + +const parseIlstItem = ( + data: Uint8Array, + itemStart: number, + itemEnd: number, + keysMap: Map, + metadata: ComfyMetadata +) => { + if (itemStart + SIZES.HEADER > itemEnd) return + + const itemIndex = readUint32(data, itemStart + 4) + const keyName = keysMap.get(itemIndex) + if (!keyName) return + + const dataBox = findIsobmffBoxByType( + data, + itemStart + SIZES.HEADER, + itemEnd, + BOX_TYPES.DATA + ) + if (dataBox) { + const value = extractMetadataValueFromDataBox( + data, + dataBox.start, + dataBox.end, + keyName + ) + if (value !== null) { + metadata[keyName.toLowerCase() as keyof ComfyMetadata] = value + } + } +} + +const parseIlstBox = ( + data: Uint8Array, + ilstStart: number, + ilstEnd: number, + keysMap: Map, + metadata: ComfyMetadata +) => { + let pos = ilstStart + while (pos < ilstEnd - SIZES.HEADER) { + const itemSize = readUint32(data, pos) + if (itemSize <= SIZES.HEADER || pos + itemSize > ilstEnd) break // Invalid item size + parseIlstItem(data, pos, pos + itemSize, keysMap, metadata) + pos += itemSize + } +} + +const findUserDataBox = (data: Uint8Array): IsobmffBoxContentRange => { + let userDataBox: IsobmffBoxContentRange = null + + // Metadata can be in 'udta' at top level or inside 'moov' + userDataBox = findIsobmffBoxByType(data, 0, data.length, BOX_TYPES.USER_DATA) + + if (!userDataBox) { + const moovBox = findIsobmffBoxByType(data, 0, data.length, BOX_TYPES.MOVIE) + if (moovBox) { + userDataBox = findIsobmffBoxByType( + data, + moovBox.start, + moovBox.end, + BOX_TYPES.USER_DATA + ) + } + } + return userDataBox +} + +const parseIsobmffMetadata = (data: Uint8Array): ComfyMetadata => { + const metadata: ComfyMetadata = {} + const userDataBox = findUserDataBox(data) + if (!userDataBox) return metadata + + const metaBox = findIsobmffBoxByType( + data, + userDataBox.start, + userDataBox.end, + BOX_TYPES.META_DATA + ) + if (!metaBox) return metadata + + const metaContentStart = metaBox.start + SIZES.VERSION + const keysBox = findIsobmffBoxByType( + data, + metaContentStart, + metaBox.end, + BOX_TYPES.KEYS + ) + if (!keysBox) return metadata + + const keysMap = parseKeysBox(data, keysBox.start, keysBox.end) + if (keysMap.size === 0) return metadata // keys box is empty or failed to parse + + const ilstBox = findIsobmffBoxByType( + data, + metaContentStart, + metaBox.end, + BOX_TYPES.ITEM_LIST + ) + if (!ilstBox) return metadata + + parseIlstBox(data, ilstBox.start, ilstBox.end, keysMap, metadata) + + return metadata +} + +/** + * Extracts ComfyUI Workflow metadata from an ISO Base Media File Format (ISOBMFF) file + * (e.g., MP4, MOV) by parsing the `udta.meta.keys` and `udta.meta.ilst` boxes. + * @param file - The file to extract metadata from. + */ +export function getFromIsobmffFile(file: File): Promise { + return new Promise((resolve) => { + const reader = new FileReader() + reader.onload = (event: ProgressEvent) => { + if (!event.target?.result) { + resolve({}) + return + } + + try { + const data = new Uint8Array(event.target.result as ArrayBuffer) + resolve(parseIsobmffMetadata(data)) + } catch (e) { + console.error('Parser: Error parsing ISOBMFF metadata:', e) + resolve({}) + } + } + reader.onerror = (err) => { + console.error('FileReader: Error reading ISOBMFF file:', err) + resolve({}) + } + reader.readAsArrayBuffer(file.slice(0, MAX_READ_BYTES)) + }) +} diff --git a/src/types/metadataTypes.ts b/src/types/metadataTypes.ts index 27449a8d5..9170ef6d4 100644 --- a/src/types/metadataTypes.ts +++ b/src/types/metadataTypes.ts @@ -46,7 +46,8 @@ export type TextRange = { export enum ASCII { GLTF = 0x46546c67, - JSON = 0x4e4f534a + JSON = 0x4e4f534a, + OPEN_BRACE = 0x7b } export enum GltfSizeBytes { @@ -78,3 +79,9 @@ export type GltfJsonData = { } [key: string]: any } + +/** + * Represents the content range [start, end) of an ISOBMFF box, excluding its header. + * Null if the box was not found. + */ +export type IsobmffBoxContentRange = { start: number; end: number } | null