feat: support video file drag-drop and paste (#9154)

src/composables/usePaste.test.ts

@@ -7,13 +7,20 @@ import type {
 } from '@/lib/litegraph/src/litegraph'
 import { app } from '@/scripts/app'
 import { createMockLGraphNode } from '@/utils/__tests__/litegraphTestUtils'
-import { createNode, isAudioNode, isImageNode } from '@/utils/litegraphUtil'
+import {
+  createNode,
+  isAudioNode,
+  isImageNode,
+  isVideoNode
+} from '@/utils/litegraphUtil'
 import {
   cloneDataTransfer,
   pasteAudioNode,
   pasteAudioNodes,
   pasteImageNode,
   pasteImageNodes,
+  pasteVideoNode,
+  pasteVideoNodes,
   usePaste
 } from './usePaste'
 
@@ -39,6 +46,13 @@ function createAudioFile(
   return new File([''], name, { type })
 }
 
+function createVideoFile(
+  name: string = 'test.mp4',
+  type: string = 'video/mp4'
+): File {
+  return new File([''], name, { type })
+}
+
 function createDataTransfer(files: File[] = []): DataTransfer {
   const dataTransfer = new DataTransfer()
   files.forEach((file) => dataTransfer.items.add(file))
@@ -300,6 +314,102 @@ describe('pasteAudioNodes', () => {
   })
 })
 
+describe('pasteVideoNode', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('should create new LoadVideo node when no video node provided', async () => {
+    const mockNode = createMockNode()
+    vi.mocked(createNode).mockResolvedValue(mockNode)
+
+    const file = createVideoFile()
+    const dataTransfer = createDataTransfer([file])
+
+    await pasteVideoNode(mockCanvas, dataTransfer.items)
+
+    expect(createNode).toHaveBeenCalledWith(mockCanvas, 'LoadVideo')
+    expect(mockNode.pasteFile).toHaveBeenCalledWith(file)
+  })
+
+  it('should use existing video node when provided', async () => {
+    const mockNode = createMockNode()
+    const file = createVideoFile()
+    const dataTransfer = createDataTransfer([file])
+
+    await pasteVideoNode(mockCanvas, dataTransfer.items, mockNode)
+
+    expect(createNode).not.toHaveBeenCalled()
+    expect(mockNode.pasteFile).toHaveBeenCalledWith(file)
+  })
+
+  it('should filter non-video items', async () => {
+    const mockNode = createMockNode()
+    const videoFile = createVideoFile()
+    const imageFile = createImageFile()
+    const dataTransfer = createDataTransfer([imageFile, videoFile])
+
+    await pasteVideoNode(mockCanvas, dataTransfer.items, mockNode)
+
+    expect(mockNode.pasteFile).toHaveBeenCalledWith(videoFile)
+    expect(mockNode.pasteFiles).toHaveBeenCalledWith([videoFile])
+  })
+
+  it('should do nothing when no video files present', async () => {
+    const mockNode = createMockNode()
+    const dataTransfer = createDataTransfer()
+
+    await pasteVideoNode(mockCanvas, dataTransfer.items, mockNode)
+
+    expect(mockNode.pasteFile).not.toHaveBeenCalled()
+    expect(mockNode.pasteFiles).not.toHaveBeenCalled()
+  })
+})
+
+describe('pasteVideoNodes', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('should create multiple nodes for multiple video files', async () => {
+    const mockNode1 = createMockNode()
+    const mockNode2 = createMockNode()
+    vi.mocked(createNode)
+      .mockResolvedValueOnce(mockNode1)
+      .mockResolvedValueOnce(mockNode2)
+
+    const file1 = createVideoFile('file1.mp4')
+    const file2 = createVideoFile('file2.webm', 'video/webm')
+
+    const result = await pasteVideoNodes(mockCanvas, [file1, file2])
+
+    expect(createNode).toHaveBeenCalledTimes(2)
+    expect(createNode).toHaveBeenNthCalledWith(1, mockCanvas, 'LoadVideo')
+    expect(createNode).toHaveBeenNthCalledWith(2, mockCanvas, 'LoadVideo')
+    expect(mockNode1.pasteFile).toHaveBeenCalledWith(file1)
+    expect(mockNode2.pasteFile).toHaveBeenCalledWith(file2)
+    expect(result).toEqual([mockNode1, mockNode2])
+  })
+
+  it('should handle empty file list', async () => {
+    const result = await pasteVideoNodes(mockCanvas, [])
+
+    expect(createNode).not.toHaveBeenCalled()
+    expect(result).toEqual([])
+  })
+
+  it('should handle single video file', async () => {
+    const mockNode = createMockNode()
+    vi.mocked(createNode).mockResolvedValue(mockNode)
+
+    const file = createVideoFile()
+    const result = await pasteVideoNodes(mockCanvas, [file])
+
+    expect(createNode).toHaveBeenCalledTimes(1)
+    expect(result).toEqual([mockNode])
+  })
+})
+
 describe('usePaste', () => {
   beforeEach(() => {
     vi.clearAllMocks()
@@ -366,6 +476,45 @@ describe('usePaste', () => {
     })
   })
 
+  it('should handle video paste', async () => {
+    const mockNode = createMockNode()
+    vi.mocked(createNode).mockResolvedValue(mockNode)
+
+    usePaste()
+
+    const file = createVideoFile()
+    const dataTransfer = createDataTransfer([file])
+    const event = new ClipboardEvent('paste', { clipboardData: dataTransfer })
+    document.dispatchEvent(event)
+
+    await vi.waitFor(() => {
+      expect(createNode).toHaveBeenCalledWith(mockCanvas, 'LoadVideo')
+      expect(mockNode.pasteFile).toHaveBeenCalledWith(file)
+    })
+  })
+
+  it('should paste video onto selected LoadVideo node', async () => {
+    const mockNode = createMockLGraphNode({
+      is_selected: true,
+      pasteFile: vi.fn(),
+      pasteFiles: vi.fn()
+    })
+    mockCanvas.current_node = mockNode
+    vi.mocked(isVideoNode).mockReturnValue(true)
+
+    usePaste()
+
+    const file = createVideoFile()
+    const dataTransfer = createDataTransfer([file])
+    const event = new ClipboardEvent('paste', { clipboardData: dataTransfer })
+    document.dispatchEvent(event)
+
+    await vi.waitFor(() => {
+      expect(createNode).not.toHaveBeenCalled()
+      expect(mockNode.pasteFile).toHaveBeenCalledWith(file)
+    })
+  })
+
   it('should handle workflow JSON paste', async () => {
     const workflow = { version: '1.0', nodes: [], extra: {} }
 

src/composables/usePaste.ts

@@ -143,6 +143,37 @@ export async function pasteAudioNodes(
   return nodes
 }
 
+export async function pasteVideoNode(
+  canvas: LGraphCanvas,
+  items: DataTransferItemList,
+  videoNode: LGraphNode | null = null
+): Promise<LGraphNode | null> {
+  if (!videoNode) {
+    videoNode = await createNode(canvas, 'LoadVideo')
+  }
+  pasteItemsOnNode(items, videoNode, 'video')
+  return videoNode
+}
+
+export async function pasteVideoNodes(
+  canvas: LGraphCanvas,
+  fileList: File[]
+): Promise<LGraphNode[]> {
+  const nodes: LGraphNode[] = []
+
+  for (const file of fileList) {
+    const transfer = new DataTransfer()
+    transfer.items.add(file)
+    const node = await pasteVideoNode(canvas, transfer.items)
+
+    if (node) {
+      nodes.push(node)
+    }
+  }
+
+  return nodes
+}
+
 /**
  * Adds a handler on paste that extracts and loads images or workflows from pasted JSON data
  */
@@ -191,13 +222,8 @@ export const usePaste = () => {
         await pasteImageNode(canvas as LGraphCanvas, items, imageNode)
         return
       } else if (item.type.startsWith('video/')) {
-        if (!videoNode) {
-          // No video node selected: add a new one
-          // TODO: when video node exists
-        } else {
-          pasteItemsOnNode(items, videoNode, 'video')
-          return
-        }
+        await pasteVideoNode(canvas as LGraphCanvas, items, videoNode)
+        return
       } else if (item.type.startsWith('audio/')) {
         await pasteAudioNode(canvas as LGraphCanvas, items, audioNode)
         return

src/scripts/app.test.ts

@@ -7,7 +7,14 @@ import type {
 } from '@/lib/litegraph/src/litegraph'
 import { ComfyApp } from './app'
 import { createNode } from '@/utils/litegraphUtil'
-import { pasteImageNode, pasteImageNodes } from '@/composables/usePaste'
+import {
+  pasteAudioNode,
+  pasteAudioNodes,
+  pasteImageNode,
+  pasteImageNodes,
+  pasteVideoNode,
+  pasteVideoNodes
+} from '@/composables/usePaste'
 import { getWorkflowDataFromFile } from '@/scripts/metadata/parser'
 
 vi.mock('@/utils/litegraphUtil', () => ({
@@ -20,8 +27,12 @@ vi.mock('@/utils/litegraphUtil', () => ({
 }))
 
 vi.mock('@/composables/usePaste', () => ({
+  pasteAudioNode: vi.fn(),
+  pasteAudioNodes: vi.fn(),
   pasteImageNode: vi.fn(),
-  pasteImageNodes: vi.fn()
+  pasteImageNodes: vi.fn(),
+  pasteVideoNode: vi.fn(),
+  pasteVideoNodes: vi.fn()
 }))
 
 vi.mock('@/scripts/metadata/parser', () => ({
@@ -130,6 +141,60 @@ describe('ComfyApp', () => {
     })
   })
 
+  describe('handleAudioFileList', () => {
+    it('should create audio nodes and select them', async () => {
+      const mockNode1 = createMockNode({ id: 1, type: 'LoadAudio' })
+      const mockNode2 = createMockNode({ id: 2, type: 'LoadAudio' })
+      vi.mocked(pasteAudioNodes).mockResolvedValue([mockNode1, mockNode2])
+
+      const file1 = createTestFile('test1.mp3', 'audio/mpeg')
+      const file2 = createTestFile('test2.wav', 'audio/wav')
+
+      await app.handleAudioFileList([file1, file2])
+
+      expect(pasteAudioNodes).toHaveBeenCalledWith(mockCanvas, [file1, file2])
+      expect(mockCanvas.selectItems).toHaveBeenCalledWith([
+        mockNode1,
+        mockNode2
+      ])
+    })
+
+    it('should not select when no nodes created', async () => {
+      vi.mocked(pasteAudioNodes).mockResolvedValue([])
+
+      await app.handleAudioFileList([createTestFile('test.mp3', 'audio/mpeg')])
+
+      expect(mockCanvas.selectItems).not.toHaveBeenCalled()
+    })
+  })
+
+  describe('handleVideoFileList', () => {
+    it('should create video nodes and select them', async () => {
+      const mockNode1 = createMockNode({ id: 1, type: 'LoadVideo' })
+      const mockNode2 = createMockNode({ id: 2, type: 'LoadVideo' })
+      vi.mocked(pasteVideoNodes).mockResolvedValue([mockNode1, mockNode2])
+
+      const file1 = createTestFile('test1.mp4', 'video/mp4')
+      const file2 = createTestFile('test2.webm', 'video/webm')
+
+      await app.handleVideoFileList([file1, file2])
+
+      expect(pasteVideoNodes).toHaveBeenCalledWith(mockCanvas, [file1, file2])
+      expect(mockCanvas.selectItems).toHaveBeenCalledWith([
+        mockNode1,
+        mockNode2
+      ])
+    })
+
+    it('should not select when no nodes created', async () => {
+      vi.mocked(pasteVideoNodes).mockResolvedValue([])
+
+      await app.handleVideoFileList([createTestFile('test.mp4', 'video/mp4')])
+
+      expect(mockCanvas.selectItems).not.toHaveBeenCalled()
+    })
+  })
+
   describe('positionBatchNodes', () => {
     it('should position batch node to the right of first node', () => {
       const mockNode1 = createMockNode({
@@ -191,6 +256,42 @@ describe('ComfyApp', () => {
       )
     })
 
+    it('should handle audio files by creating LoadAudio node', async () => {
+      vi.mocked(getWorkflowDataFromFile).mockResolvedValue({})
+
+      const mockNode = createMockNode({ type: 'LoadAudio' })
+      vi.mocked(createNode).mockResolvedValue(mockNode)
+
+      const audioFile = createTestFile('test.mp3', 'audio/mpeg')
+
+      await app.handleFile(audioFile)
+
+      expect(createNode).toHaveBeenCalledWith(mockCanvas, 'LoadAudio')
+      expect(pasteAudioNode).toHaveBeenCalledWith(
+        mockCanvas,
+        expect.any(DataTransferItemList),
+        mockNode
+      )
+    })
+
+    it('should handle video files by creating LoadVideo node', async () => {
+      vi.mocked(getWorkflowDataFromFile).mockResolvedValue({})
+
+      const mockNode = createMockNode({ type: 'LoadVideo' })
+      vi.mocked(createNode).mockResolvedValue(mockNode)
+
+      const videoFile = createTestFile('test.mp4', 'video/mp4')
+
+      await app.handleFile(videoFile)
+
+      expect(createNode).toHaveBeenCalledWith(mockCanvas, 'LoadVideo')
+      expect(pasteVideoNode).toHaveBeenCalledWith(
+        mockCanvas,
+        expect.any(DataTransferItemList),
+        mockNode
+      )
+    })
+
     it('should handle image files with non-workflow metadata by creating LoadImage node', async () => {
       vi.mocked(getWorkflowDataFromFile).mockResolvedValue({
         Software: 'gnome-screenshot'

src/scripts/app.ts

@@ -114,14 +114,18 @@ import { ensureCorrectLayoutScale } from '@/renderer/extensions/vueNodes/layout/
 import {
   extractFilesFromDragEvent,
   hasAudioType,
-  hasImageType
+  hasImageType,
+  hasVideoType,
+  isMediaFile
 } from '@/utils/eventUtils'
 import { getWorkflowDataFromFile } from '@/scripts/metadata/parser'
 import {
   pasteAudioNode,
   pasteAudioNodes,
   pasteImageNode,
-  pasteImageNodes
+  pasteImageNodes,
+  pasteVideoNode,
+  pasteVideoNodes
 } from '@/composables/usePaste'
 
 export const ANIM_PREVIEW_WIDGET = '$$comfy_animation_preview'
@@ -571,7 +575,9 @@ export class ComfyApp {
           workspace.spinner = true
           const imageFiles = files.filter(hasImageType)
           const audioFiles = files.filter(hasAudioType)
-          const totalMedia = imageFiles.length + audioFiles.length
+          const videoFiles = files.filter(hasVideoType)
+          const totalMedia =
+            imageFiles.length + audioFiles.length + videoFiles.length
           const hasMultipleMedia = totalMedia > 1
 
           if (hasMultipleMedia) {
@@ -581,8 +587,10 @@ export class ComfyApp {
             if (audioFiles.length > 0) {
               await this.handleAudioFileList(audioFiles)
             }
-            const handled = new Set([...imageFiles, ...audioFiles])
-            for (const file of files.filter((f) => !handled.has(f))) {
+            if (videoFiles.length > 0) {
+              await this.handleVideoFileList(videoFiles)
+            }
+            for (const file of files.filter((f) => !isMediaFile(f))) {
               await this.handleFile(file, 'file_drop', {
                 deferWarnings: true
               })
@@ -1581,17 +1589,21 @@ export class ComfyApp {
     const { workflow, prompt, parameters, templates } = workflowData ?? {}
 
     if (!(workflow || prompt || parameters || templates)) {
-      if (file.type.startsWith('image')) {
+      const mediaNodeTypes: Record<string, [string, typeof pasteImageNode]> = {
+        image: ['LoadImage', pasteImageNode],
+        audio: ['LoadAudio', pasteAudioNode],
+        video: ['LoadVideo', pasteVideoNode]
+      }
+
+      const mediaType = Object.keys(mediaNodeTypes).find((t) =>
+        file.type.startsWith(t)
+      )
+      if (mediaType) {
+        const [nodeType, pasteFn] = mediaNodeTypes[mediaType]
         const transfer = new DataTransfer()
         transfer.items.add(file)
-        const imageNode = await createNode(this.canvas, 'LoadImage')
-        await pasteImageNode(this.canvas, transfer.items, imageNode)
-        return
-      } else if (file.type.startsWith('audio')) {
-        const transfer = new DataTransfer()
-        transfer.items.add(file)
-        const audioNode = await createNode(this.canvas, 'LoadAudio')
-        await pasteAudioNode(this.canvas, transfer.items, audioNode)
+        const node = await createNode(this.canvas, nodeType)
+        await pasteFn(this.canvas, transfer.items, node)
         return
       }
 
@@ -1703,6 +1715,14 @@ export class ComfyApp {
     this.canvas.selectItems(audioNodes)
   }
 
+  async handleVideoFileList(fileList: File[]) {
+    const videoNodes = await pasteVideoNodes(this.canvas, fileList)
+    if (videoNodes.length === 0) return
+
+    this.positionNodes(videoNodes)
+    this.canvas.selectItems(videoNodes)
+  }
+
   /**
    * Positions batched nodes in drag and drop
    * @param nodes

src/utils/eventUtils.test.ts

@@ -1,5 +1,10 @@
 import { describe, expect, it } from 'vitest'
-import { hasAudioType, hasImageType } from './eventUtils'
+import {
+  hasAudioType,
+  hasImageType,
+  hasVideoType,
+  isMediaFile
+} from './eventUtils'
 
 describe('hasImageType', () => {
   it('should return true for image types', () => {
@@ -24,3 +29,28 @@ describe('hasAudioType', () => {
     expect(hasAudioType({ type: 'video/mp4' } as File)).toBe(false)
   })
 })
+
+describe('hasVideoType', () => {
+  it('should return true for video types', () => {
+    expect(hasVideoType({ type: 'video/mp4' } as File)).toBe(true)
+    expect(hasVideoType({ type: 'video/webm' } as File)).toBe(true)
+  })
+
+  it('should return false for non-video types', () => {
+    expect(hasVideoType({ type: 'audio/mpeg' } as File)).toBe(false)
+    expect(hasVideoType({ type: 'image/png' } as File)).toBe(false)
+  })
+})
+
+describe('isMediaFile', () => {
+  it('should return true for image, audio, and video types', () => {
+    expect(isMediaFile({ type: 'image/png' } as File)).toBe(true)
+    expect(isMediaFile({ type: 'audio/mpeg' } as File)).toBe(true)
+    expect(isMediaFile({ type: 'video/mp4' } as File)).toBe(true)
+  })
+
+  it('should return false for non-media types', () => {
+    expect(isMediaFile({ type: 'text/plain' } as File)).toBe(false)
+    expect(isMediaFile({ type: 'application/json' } as File)).toBe(false)
+  })
+})

src/utils/eventUtils.ts

@@ -32,3 +32,11 @@ export function hasImageType({ type }: File): boolean {
 export function hasAudioType({ type }: File): boolean {
   return type.startsWith('audio')
 }
+
+export function hasVideoType({ type }: File): boolean {
+  return type.startsWith('video')
+}
+
+export function isMediaFile(file: File): boolean {
+  return hasImageType(file) || hasAudioType(file) || hasVideoType(file)
+}