mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-05-21 21:09:00 +00:00
[backport cloud/1.44] fix: handle annotated output media paths in missing media scan (#12122)
Backport of #12069 to `cloud/1.44` Automatically created by backport workflow. ┆Issue is synchronized with this [Notion page](https://www.notion.so/PR-12122-backport-cloud-1-44-fix-handle-annotated-output-media-paths-in-missing-media-scan-35d6d73d36508174bfc1f2590edd1a03) by [Unito](https://www.unito.io) Co-authored-by: jaeone94 <89377375+jaeone94@users.noreply.github.com>
This commit is contained in:
@@ -3,12 +3,14 @@ import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
appendWorkflowJsonExt,
|
||||
ensureWorkflowSuffix,
|
||||
getFilePathSeparatorVariants,
|
||||
getFilenameDetails,
|
||||
getMediaTypeFromFilename,
|
||||
getPathDetails,
|
||||
highlightQuery,
|
||||
isCivitaiModelUrl,
|
||||
isPreviewableMediaType,
|
||||
joinFilePath,
|
||||
truncateFilename
|
||||
} from './formatUtil'
|
||||
|
||||
@@ -299,6 +301,42 @@ describe('formatUtil', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('joinFilePath', () => {
|
||||
it('joins subfolder and filename with normalized slash separators', () => {
|
||||
expect(joinFilePath('nested\\folder', 'child\\file.png')).toBe(
|
||||
'nested/folder/child/file.png'
|
||||
)
|
||||
})
|
||||
|
||||
it('trims boundary separators without changing the filename body', () => {
|
||||
expect(joinFilePath('/nested/folder/', '/file.png')).toBe(
|
||||
'nested/folder/file.png'
|
||||
)
|
||||
})
|
||||
|
||||
it('returns the normalized filename when no subfolder is provided', () => {
|
||||
expect(joinFilePath('', 'nested\\file.png')).toBe('nested/file.png')
|
||||
})
|
||||
|
||||
it('returns the normalized subfolder without a trailing slash when no filename is provided', () => {
|
||||
expect(joinFilePath('nested\\folder', '')).toBe('nested/folder')
|
||||
expect(joinFilePath('nested\\folder', null)).toBe('nested/folder')
|
||||
})
|
||||
})
|
||||
|
||||
describe('getFilePathSeparatorVariants', () => {
|
||||
it('returns slash and backslash variants for nested paths', () => {
|
||||
expect(getFilePathSeparatorVariants('nested\\folder/file.png')).toEqual([
|
||||
'nested/folder/file.png',
|
||||
'nested\\folder\\file.png'
|
||||
])
|
||||
})
|
||||
|
||||
it('returns a single value when no separator is present', () => {
|
||||
expect(getFilePathSeparatorVariants('file.png')).toEqual(['file.png'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('appendWorkflowJsonExt', () => {
|
||||
it('appends .app.json when isApp is true', () => {
|
||||
expect(appendWorkflowJsonExt('test', true)).toBe('test.app.json')
|
||||
|
||||
@@ -256,6 +256,31 @@ export function isValidUrl(url: string): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
export function joinFilePath(
|
||||
subfolder: string | null | undefined,
|
||||
filename: string | null | undefined
|
||||
): string {
|
||||
const normalizedSubfolder = normalizeFilePathSeparators(
|
||||
subfolder ?? ''
|
||||
).replace(/^\/+|\/+$/g, '')
|
||||
const normalizedFilename = normalizeFilePathSeparators(
|
||||
filename ?? ''
|
||||
).replace(/^\/+/g, '')
|
||||
if (!normalizedSubfolder) return normalizedFilename
|
||||
if (!normalizedFilename) return normalizedSubfolder
|
||||
return `${normalizedSubfolder}/${normalizedFilename}`
|
||||
}
|
||||
|
||||
export function getFilePathSeparatorVariants(filepath: string): string[] {
|
||||
const slashPath = normalizeFilePathSeparators(filepath)
|
||||
const backslashPath = slashPath.replace(/\//g, '\\')
|
||||
return slashPath === backslashPath ? [slashPath] : [slashPath, backslashPath]
|
||||
}
|
||||
|
||||
function normalizeFilePathSeparators(filepath: string): string {
|
||||
return filepath.replace(/[\\/]+/g, '/')
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a filepath into its filename and subfolder components.
|
||||
*
|
||||
@@ -274,8 +299,7 @@ export function parseFilePath(filepath: string): {
|
||||
} {
|
||||
if (!filepath?.trim()) return { filename: '', subfolder: '' }
|
||||
|
||||
const normalizedPath = filepath
|
||||
.replace(/[\\/]+/g, '/') // Normalize path separators
|
||||
const normalizedPath = normalizeFilePathSeparators(filepath)
|
||||
.replace(/^\//, '') // Remove leading slash
|
||||
.replace(/\/$/, '') // Remove trailing slash
|
||||
|
||||
|
||||
@@ -543,7 +543,7 @@ describe('realtime scan verifies pending cloud candidates', () => {
|
||||
}
|
||||
])
|
||||
const verifySpy = vi
|
||||
.spyOn(missingMediaScan, 'verifyCloudMediaCandidates')
|
||||
.spyOn(missingMediaScan, 'verifyMediaCandidates')
|
||||
.mockImplementation(async (candidates) => {
|
||||
for (const c of candidates) c.isMissing = true
|
||||
})
|
||||
@@ -686,7 +686,7 @@ describe('realtime verification staleness guards', () => {
|
||||
let resolveVerify: (() => void) | undefined
|
||||
const verifyPromise = new Promise<void>((r) => (resolveVerify = r))
|
||||
const verifySpy = vi
|
||||
.spyOn(missingMediaScan, 'verifyCloudMediaCandidates')
|
||||
.spyOn(missingMediaScan, 'verifyMediaCandidates')
|
||||
.mockImplementation(async (candidates) => {
|
||||
await verifyPromise
|
||||
for (const c of candidates) c.isMissing = true
|
||||
|
||||
@@ -28,7 +28,7 @@ import {
|
||||
import { useMissingModelStore } from '@/platform/missingModel/missingModelStore'
|
||||
import {
|
||||
scanNodeMediaCandidates,
|
||||
verifyCloudMediaCandidates
|
||||
verifyMediaCandidates
|
||||
} from '@/platform/missingMedia/missingMediaScan'
|
||||
import { useMissingMediaStore } from '@/platform/missingMedia/missingMediaStore'
|
||||
import { useMissingNodesErrorStore } from '@/platform/nodeReplacement/missingNodesErrorStore'
|
||||
@@ -209,8 +209,8 @@ function scanSingleNodeErrors(node: LGraphNode): void {
|
||||
if (confirmedMedia.length) {
|
||||
useMissingMediaStore().addMissingMedia(confirmedMedia)
|
||||
}
|
||||
// Cloud media scans always return isMissing: undefined pending
|
||||
// verification against the input-assets list.
|
||||
// Cloud media scans return pending for asset verification. OSS scans only
|
||||
// return pending for generated output/temp media.
|
||||
const pendingMedia = mediaCandidates.filter((c) => c.isMissing === undefined)
|
||||
if (pendingMedia.length) {
|
||||
void verifyAndAddPendingMedia(pendingMedia)
|
||||
@@ -282,7 +282,7 @@ async function verifyAndAddPendingMedia(
|
||||
): Promise<void> {
|
||||
const rootGraphAtScan = app.rootGraph
|
||||
try {
|
||||
await verifyCloudMediaCandidates(pending)
|
||||
await verifyMediaCandidates(pending, { isCloud })
|
||||
if (app.rootGraph !== rootGraphAtScan) return
|
||||
const verified = pending.filter(
|
||||
(c) => c.isMissing === true && isCandidateStillActive(c.nodeId)
|
||||
|
||||
@@ -480,12 +480,27 @@ function createAssetService() {
|
||||
includePublic: boolean = true,
|
||||
{ limit = DEFAULT_LIMIT, offset = 0, signal }: AssetPaginationOptions = {}
|
||||
): Promise<AssetItem[]> {
|
||||
const data = await handleAssetRequest(
|
||||
const data = await getAssetsPageByTag(tag, includePublic, {
|
||||
limit,
|
||||
offset,
|
||||
signal
|
||||
})
|
||||
|
||||
return data.assets
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets one paginated asset response filtered by a specific tag.
|
||||
*/
|
||||
async function getAssetsPageByTag(
|
||||
tag: string,
|
||||
includePublic: boolean = true,
|
||||
{ limit = DEFAULT_LIMIT, offset = 0, signal }: AssetPaginationOptions = {}
|
||||
): Promise<AssetResponse> {
|
||||
return await handleAssetRequest(
|
||||
{ includeTags: [tag], limit, offset, includePublic, signal },
|
||||
`assets for tag ${tag}`
|
||||
)
|
||||
|
||||
return data.assets
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -511,16 +526,11 @@ function createAssetService() {
|
||||
while (true) {
|
||||
if (signal?.aborted) throw createAbortError()
|
||||
|
||||
const data = await handleAssetRequest(
|
||||
{
|
||||
includeTags: [tag],
|
||||
limit: pageSize,
|
||||
offset,
|
||||
includePublic,
|
||||
signal
|
||||
},
|
||||
`assets for tag ${tag}`
|
||||
)
|
||||
const data = await getAssetsPageByTag(tag, includePublic, {
|
||||
limit: pageSize,
|
||||
offset,
|
||||
signal
|
||||
})
|
||||
const batch = data.assets
|
||||
if (batch.length === 0) {
|
||||
return assets
|
||||
@@ -935,6 +945,7 @@ function createAssetService() {
|
||||
getAssetsForNodeType,
|
||||
getAssetDetails,
|
||||
getAssetsByTag,
|
||||
getAssetsPageByTag,
|
||||
getAllAssetsByTag,
|
||||
getInputAssetsIncludingPublic,
|
||||
invalidateInputAssetsIncludingPublic,
|
||||
|
||||
80
src/platform/missingMedia/mediaPathDetectionUtil.test.ts
Normal file
80
src/platform/missingMedia/mediaPathDetectionUtil.test.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import {
|
||||
getAnnotatedMediaPathTypeForDetection,
|
||||
getMediaPathDetectionNames,
|
||||
normalizeAnnotatedMediaPathForDetection
|
||||
} from './mediaPathDetectionUtil'
|
||||
|
||||
describe('normalizeAnnotatedMediaPathForDetection', () => {
|
||||
it.each([
|
||||
['photo.png [input]', 'photo.png'],
|
||||
['result.png [output]', 'result.png'],
|
||||
['photo.png [input]', 'photo.png'],
|
||||
['with spaces.png [output]', 'with spaces.png'],
|
||||
['nested/folder/video.mp4 [output]', 'nested/folder/video.mp4']
|
||||
])('strips Core-style annotation from %s', (value, expected) => {
|
||||
expect(normalizeAnnotatedMediaPathForDetection(value)).toBe(expected)
|
||||
})
|
||||
|
||||
it.each([
|
||||
['photo.png[input]', 'photo.png'],
|
||||
['result.png[output]', 'result.png'],
|
||||
['with spaces.png [output]', 'with spaces.png']
|
||||
])('strips Cloud compact annotation from %s', (value, expected) => {
|
||||
expect(
|
||||
normalizeAnnotatedMediaPathForDetection(value, {
|
||||
allowCompactSuffix: true
|
||||
})
|
||||
).toBe(expected)
|
||||
})
|
||||
|
||||
it('does not strip compact annotations in Core mode', () => {
|
||||
expect(normalizeAnnotatedMediaPathForDetection('photo.png[input]')).toBe(
|
||||
'photo.png[input]'
|
||||
)
|
||||
})
|
||||
|
||||
it.each(['photo.png [draft]', 'photo [output] copy.png', 'photo.png', ''])(
|
||||
'leaves non-matching values unchanged: %s',
|
||||
(value) => {
|
||||
expect(normalizeAnnotatedMediaPathForDetection(value)).toBe(value)
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
describe('getMediaPathDetectionNames', () => {
|
||||
it('returns raw and normalized names when an annotation is stripped', () => {
|
||||
expect(getMediaPathDetectionNames('photo.png [input]')).toEqual([
|
||||
'photo.png [input]',
|
||||
'photo.png'
|
||||
])
|
||||
})
|
||||
|
||||
it('returns only the raw name when no annotation is stripped', () => {
|
||||
expect(getMediaPathDetectionNames('photo.png')).toEqual(['photo.png'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('getAnnotatedMediaPathTypeForDetection', () => {
|
||||
it.each([
|
||||
['photo.png [input]', 'input'],
|
||||
['photo.png [output]', 'output']
|
||||
])('returns the Core-style annotation type from %s', (value, expected) => {
|
||||
expect(getAnnotatedMediaPathTypeForDetection(value)).toBe(expected)
|
||||
})
|
||||
|
||||
it('returns the compact annotation type in Cloud mode', () => {
|
||||
expect(
|
||||
getAnnotatedMediaPathTypeForDetection('photo.png[output]', {
|
||||
allowCompactSuffix: true
|
||||
})
|
||||
).toBe('output')
|
||||
})
|
||||
|
||||
it('returns undefined when no supported annotation is present', () => {
|
||||
expect(getAnnotatedMediaPathTypeForDetection('photo.png [draft]')).toBe(
|
||||
undefined
|
||||
)
|
||||
})
|
||||
})
|
||||
44
src/platform/missingMedia/mediaPathDetectionUtil.ts
Normal file
44
src/platform/missingMedia/mediaPathDetectionUtil.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
// Missing-media-scoped helpers for deriving comparison keys from media widget paths.
|
||||
const CORE_ANNOTATED_MEDIA_PATTERN = /\s+\[(input|output)\]$/
|
||||
const CLOUD_ANNOTATED_MEDIA_PATTERN = /\s*\[(input|output)\]$/
|
||||
|
||||
type AnnotatedMediaPathType = 'input' | 'output'
|
||||
|
||||
interface AnnotatedMediaPathOptions {
|
||||
allowCompactSuffix?: boolean
|
||||
}
|
||||
|
||||
function getAnnotatedMediaPathMatch(
|
||||
value: string,
|
||||
options: AnnotatedMediaPathOptions = {}
|
||||
): RegExpMatchArray | null {
|
||||
const pattern = options.allowCompactSuffix
|
||||
? CLOUD_ANNOTATED_MEDIA_PATTERN
|
||||
: CORE_ANNOTATED_MEDIA_PATTERN
|
||||
return value.match(pattern)
|
||||
}
|
||||
|
||||
export function getAnnotatedMediaPathTypeForDetection(
|
||||
value: string,
|
||||
options: AnnotatedMediaPathOptions = {}
|
||||
): AnnotatedMediaPathType | undefined {
|
||||
return getAnnotatedMediaPathMatch(value, options)?.[1] as
|
||||
| AnnotatedMediaPathType
|
||||
| undefined
|
||||
}
|
||||
|
||||
export function normalizeAnnotatedMediaPathForDetection(
|
||||
value: string,
|
||||
options: AnnotatedMediaPathOptions = {}
|
||||
): string {
|
||||
const match = getAnnotatedMediaPathMatch(value, options)
|
||||
return match ? value.slice(0, match.index) : value
|
||||
}
|
||||
|
||||
export function getMediaPathDetectionNames(
|
||||
value: string,
|
||||
options: AnnotatedMediaPathOptions = {}
|
||||
): string[] {
|
||||
const normalized = normalizeAnnotatedMediaPathForDetection(value, options)
|
||||
return normalized === value ? [value] : [value, normalized]
|
||||
}
|
||||
325
src/platform/missingMedia/missingMediaAssetResolver.test.ts
Normal file
325
src/platform/missingMedia/missingMediaAssetResolver.test.ts
Normal file
@@ -0,0 +1,325 @@
|
||||
import { fromAny } from '@total-typescript/shoehorn'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import type { AssetItem } from '@/platform/assets/schemas/assetSchema'
|
||||
import type * as AssetServiceModule from '@/platform/assets/services/assetService'
|
||||
import type * as FetchJobsModule from '@/platform/remote/comfyui/jobs/fetchJobs'
|
||||
import type { JobListItem } from '@/platform/remote/comfyui/jobs/jobTypes'
|
||||
import {
|
||||
getAssetDetectionNames,
|
||||
resolveMissingMediaAssetSources
|
||||
} from './missingMediaAssetResolver'
|
||||
|
||||
const { mockGetInputAssetsIncludingPublic, mockGetAssetsPageByTag } =
|
||||
vi.hoisted(() => ({
|
||||
mockGetInputAssetsIncludingPublic: vi.fn(),
|
||||
mockGetAssetsPageByTag: vi.fn()
|
||||
}))
|
||||
|
||||
const { mockFetchHistoryPage } = vi.hoisted(() => ({
|
||||
mockFetchHistoryPage: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('@/platform/assets/services/assetService', async () => {
|
||||
const actual = await vi.importActual<typeof AssetServiceModule>(
|
||||
'@/platform/assets/services/assetService'
|
||||
)
|
||||
|
||||
return {
|
||||
...actual,
|
||||
assetService: {
|
||||
...actual.assetService,
|
||||
getInputAssetsIncludingPublic: mockGetInputAssetsIncludingPublic,
|
||||
getAssetsPageByTag: mockGetAssetsPageByTag
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@/platform/remote/comfyui/jobs/fetchJobs', async () => {
|
||||
const actual = await vi.importActual<typeof FetchJobsModule>(
|
||||
'@/platform/remote/comfyui/jobs/fetchJobs'
|
||||
)
|
||||
|
||||
return {
|
||||
...actual,
|
||||
fetchHistoryPage: mockFetchHistoryPage
|
||||
}
|
||||
})
|
||||
|
||||
function makeAsset(name: string, assetHash: string | null = null): AssetItem {
|
||||
return {
|
||||
id: name,
|
||||
name,
|
||||
asset_hash: assetHash,
|
||||
mime_type: null,
|
||||
tags: ['input']
|
||||
}
|
||||
}
|
||||
|
||||
function makeHistoryJob(
|
||||
filename: string,
|
||||
options: { id?: string; subfolder?: string } = {}
|
||||
): JobListItem {
|
||||
return fromAny<JobListItem, unknown>({
|
||||
id: options.id ?? filename,
|
||||
status: 'completed',
|
||||
create_time: 0,
|
||||
priority: 0,
|
||||
preview_output: {
|
||||
filename,
|
||||
subfolder: options.subfolder ?? '',
|
||||
type: 'output',
|
||||
nodeId: '1',
|
||||
mediaType: 'images'
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function makeHistoryPage(
|
||||
jobs: JobListItem[],
|
||||
options: { offset?: number; hasMore?: boolean; total?: number } = {}
|
||||
) {
|
||||
return {
|
||||
jobs,
|
||||
total: options.total ?? jobs.length,
|
||||
offset: options.offset ?? 0,
|
||||
limit: 200,
|
||||
hasMore: options.hasMore ?? false
|
||||
}
|
||||
}
|
||||
|
||||
function makeAssetPage(
|
||||
assets: AssetItem[],
|
||||
options: { hasMore?: boolean; total?: number } = {}
|
||||
) {
|
||||
return {
|
||||
assets,
|
||||
total: options.total ?? assets.length,
|
||||
has_more: options.hasMore ?? false
|
||||
}
|
||||
}
|
||||
|
||||
describe('resolveMissingMediaAssetSources', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockGetInputAssetsIncludingPublic.mockResolvedValue([])
|
||||
mockGetAssetsPageByTag.mockResolvedValue(makeAssetPage([]))
|
||||
mockFetchHistoryPage.mockResolvedValue(makeHistoryPage([]))
|
||||
})
|
||||
|
||||
it('loads cloud input assets when requested', async () => {
|
||||
const inputAsset = makeAsset('photo.png')
|
||||
mockGetInputAssetsIncludingPublic.mockResolvedValue([inputAsset])
|
||||
|
||||
const result = await resolveMissingMediaAssetSources({
|
||||
isCloud: true,
|
||||
includeGeneratedAssets: false,
|
||||
generatedMatchNames: new Set(),
|
||||
allowCompactSuffix: true
|
||||
})
|
||||
|
||||
expect(result.inputAssets).toEqual([inputAsset])
|
||||
expect(result.generatedAssets).toEqual([])
|
||||
expect(mockGetInputAssetsIncludingPublic).toHaveBeenCalledWith(
|
||||
expect.any(AbortSignal)
|
||||
)
|
||||
expect(mockFetchHistoryPage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('loads cloud output assets by tag when generated candidates need verification', async () => {
|
||||
const outputAsset = makeAsset('output.png')
|
||||
mockGetAssetsPageByTag.mockResolvedValue(makeAssetPage([outputAsset]))
|
||||
|
||||
const result = await resolveMissingMediaAssetSources({
|
||||
isCloud: true,
|
||||
includeGeneratedAssets: true,
|
||||
generatedMatchNames: new Set(['output.png']),
|
||||
allowCompactSuffix: true
|
||||
})
|
||||
|
||||
expect(result.generatedAssets).toEqual([outputAsset])
|
||||
expect(mockGetAssetsPageByTag).toHaveBeenCalledWith(
|
||||
'output',
|
||||
true,
|
||||
expect.objectContaining({
|
||||
limit: 500,
|
||||
offset: 0,
|
||||
signal: expect.any(AbortSignal)
|
||||
})
|
||||
)
|
||||
expect(mockFetchHistoryPage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('stops reading cloud output asset pages once all requested names are found', async () => {
|
||||
const target = 'target-output.png'
|
||||
mockGetAssetsPageByTag.mockResolvedValueOnce(
|
||||
makeAssetPage([makeAsset(target)], { hasMore: true, total: 501 })
|
||||
)
|
||||
|
||||
const result = await resolveMissingMediaAssetSources({
|
||||
isCloud: true,
|
||||
includeGeneratedAssets: true,
|
||||
generatedMatchNames: new Set([target]),
|
||||
allowCompactSuffix: true
|
||||
})
|
||||
|
||||
expect(result.generatedAssets).toEqual([makeAsset(target)])
|
||||
expect(mockGetAssetsPageByTag).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('aborts cloud output asset loading when input asset loading fails', async () => {
|
||||
const inputError = new Error('input failed')
|
||||
let rejectInputAssets!: (err: Error) => void
|
||||
let resolveOutputAssets!: (page: ReturnType<typeof makeAssetPage>) => void
|
||||
mockGetInputAssetsIncludingPublic.mockReturnValueOnce(
|
||||
new Promise<AssetItem[]>((_, reject) => {
|
||||
rejectInputAssets = reject
|
||||
})
|
||||
)
|
||||
mockGetAssetsPageByTag.mockReturnValueOnce(
|
||||
new Promise((resolve) => {
|
||||
resolveOutputAssets = resolve
|
||||
})
|
||||
)
|
||||
|
||||
const promise = resolveMissingMediaAssetSources({
|
||||
isCloud: true,
|
||||
includeGeneratedAssets: true,
|
||||
generatedMatchNames: new Set(['target.png']),
|
||||
allowCompactSuffix: true
|
||||
})
|
||||
|
||||
await Promise.resolve()
|
||||
expect(mockGetAssetsPageByTag).toHaveBeenCalledOnce()
|
||||
|
||||
rejectInputAssets(inputError)
|
||||
await expect(promise).rejects.toBe(inputError)
|
||||
|
||||
resolveOutputAssets(makeAssetPage([makeAsset('other.png')]))
|
||||
await Promise.resolve()
|
||||
|
||||
const outputSignal = mockGetAssetsPageByTag.mock.calls[0]?.[2]?.signal
|
||||
expect(outputSignal).toBeInstanceOf(AbortSignal)
|
||||
expect(outputSignal.aborted).toBe(true)
|
||||
expect(mockFetchHistoryPage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('stops reading generated history once all requested names are found', async () => {
|
||||
const target = 'target.png'
|
||||
mockFetchHistoryPage.mockResolvedValueOnce(
|
||||
makeHistoryPage([makeHistoryJob(target)], {
|
||||
hasMore: true,
|
||||
total: 400
|
||||
})
|
||||
)
|
||||
|
||||
const result = await resolveMissingMediaAssetSources({
|
||||
isCloud: false,
|
||||
includeGeneratedAssets: true,
|
||||
generatedMatchNames: new Set([target]),
|
||||
allowCompactSuffix: true
|
||||
})
|
||||
|
||||
expect(result.generatedAssets).toHaveLength(1)
|
||||
expect(result.generatedAssets[0].name).toBe(target)
|
||||
expect(mockFetchHistoryPage).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('advances pagination from the requested offset, not the echoed offset', async () => {
|
||||
const target = 'target.png'
|
||||
mockFetchHistoryPage
|
||||
.mockResolvedValueOnce(
|
||||
makeHistoryPage(
|
||||
Array.from({ length: 200 }, (_, index) =>
|
||||
makeHistoryJob(`other-${index}.png`)
|
||||
),
|
||||
{ offset: 0, hasMore: true, total: 201 }
|
||||
)
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
makeHistoryPage([makeHistoryJob(target)], {
|
||||
offset: 0,
|
||||
hasMore: true,
|
||||
total: 201
|
||||
})
|
||||
)
|
||||
|
||||
await resolveMissingMediaAssetSources({
|
||||
isCloud: false,
|
||||
includeGeneratedAssets: true,
|
||||
generatedMatchNames: new Set([target]),
|
||||
allowCompactSuffix: true
|
||||
})
|
||||
|
||||
expect(mockFetchHistoryPage).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.any(Function),
|
||||
200,
|
||||
0
|
||||
)
|
||||
expect(mockFetchHistoryPage).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.any(Function),
|
||||
200,
|
||||
200
|
||||
)
|
||||
})
|
||||
|
||||
it('stops if history reports hasMore but returns an empty page', async () => {
|
||||
mockFetchHistoryPage.mockResolvedValueOnce(
|
||||
makeHistoryPage([], { hasMore: true, total: 1 })
|
||||
)
|
||||
|
||||
const result = await resolveMissingMediaAssetSources({
|
||||
isCloud: false,
|
||||
includeGeneratedAssets: true,
|
||||
generatedMatchNames: new Set(['missing.png']),
|
||||
allowCompactSuffix: true
|
||||
})
|
||||
|
||||
expect(result.generatedAssets).toEqual([])
|
||||
expect(mockFetchHistoryPage).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('stops if history repeats the same job page', async () => {
|
||||
const repeatedJob = makeHistoryJob('other.png', { id: 'same-job' })
|
||||
mockFetchHistoryPage
|
||||
.mockResolvedValueOnce(
|
||||
makeHistoryPage([repeatedJob], { hasMore: true, total: 2 })
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
makeHistoryPage([repeatedJob], { offset: 1, hasMore: true, total: 2 })
|
||||
)
|
||||
|
||||
const result = await resolveMissingMediaAssetSources({
|
||||
isCloud: false,
|
||||
includeGeneratedAssets: true,
|
||||
generatedMatchNames: new Set(['missing.png']),
|
||||
allowCompactSuffix: true
|
||||
})
|
||||
|
||||
expect(result.generatedAssets).toHaveLength(1)
|
||||
expect(mockFetchHistoryPage).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('includes slash and backslash subfolder identifiers for detection', () => {
|
||||
const names = getAssetDetectionNames(
|
||||
{
|
||||
...makeAsset('child\\photo.png', 'hash.png'),
|
||||
user_metadata: { subfolder: 'nested\\folder' }
|
||||
},
|
||||
{ allowCompactSuffix: true }
|
||||
)
|
||||
|
||||
expect(names).toEqual(
|
||||
expect.arrayContaining([
|
||||
'child\\photo.png',
|
||||
'hash.png',
|
||||
'nested/folder/child/photo.png',
|
||||
'nested\\folder\\child\\photo.png'
|
||||
])
|
||||
)
|
||||
expect(names).not.toContain('nested/folder/hash.png')
|
||||
expect(names).not.toContain('nested\\folder\\hash.png')
|
||||
})
|
||||
})
|
||||
286
src/platform/missingMedia/missingMediaAssetResolver.ts
Normal file
286
src/platform/missingMedia/missingMediaAssetResolver.ts
Normal file
@@ -0,0 +1,286 @@
|
||||
import type { AssetItem } from '@/platform/assets/schemas/assetSchema'
|
||||
import { assetService } from '@/platform/assets/services/assetService'
|
||||
import { fetchHistoryPage } from '@/platform/remote/comfyui/jobs/fetchJobs'
|
||||
import type { JobListItem } from '@/platform/remote/comfyui/jobs/jobTypes'
|
||||
import { api } from '@/scripts/api'
|
||||
import { getFilePathSeparatorVariants, joinFilePath } from '@/utils/formatUtil'
|
||||
import { getMediaPathDetectionNames } from './mediaPathDetectionUtil'
|
||||
|
||||
const HISTORY_MEDIA_ASSETS_PAGE_SIZE = 200
|
||||
const CLOUD_OUTPUT_ASSETS_PAGE_SIZE = 500
|
||||
|
||||
interface MediaPathDetectionOptions {
|
||||
allowCompactSuffix: boolean
|
||||
}
|
||||
|
||||
export interface MissingMediaAssetSources {
|
||||
inputAssets: AssetItem[]
|
||||
generatedAssets: AssetItem[]
|
||||
}
|
||||
|
||||
export interface ResolveMissingMediaAssetSourcesOptions {
|
||||
signal?: AbortSignal
|
||||
isCloud: boolean
|
||||
includeGeneratedAssets: boolean
|
||||
generatedMatchNames: ReadonlySet<string>
|
||||
allowCompactSuffix: boolean
|
||||
}
|
||||
|
||||
export type MissingMediaAssetResolver = (
|
||||
options: ResolveMissingMediaAssetSourcesOptions
|
||||
) => Promise<MissingMediaAssetSources>
|
||||
|
||||
export async function resolveMissingMediaAssetSources({
|
||||
signal,
|
||||
isCloud,
|
||||
includeGeneratedAssets,
|
||||
generatedMatchNames,
|
||||
allowCompactSuffix
|
||||
}: ResolveMissingMediaAssetSourcesOptions): Promise<MissingMediaAssetSources> {
|
||||
const pathOptions = { allowCompactSuffix }
|
||||
|
||||
const controller = new AbortController()
|
||||
const abortFromCaller = () => controller.abort(signal?.reason)
|
||||
if (signal?.aborted) {
|
||||
abortFromCaller()
|
||||
} else {
|
||||
signal?.addEventListener('abort', abortFromCaller, { once: true })
|
||||
}
|
||||
|
||||
try {
|
||||
const [inputAssets, generatedAssets] = await Promise.all([
|
||||
abortSiblingsOnFailure(
|
||||
isCloud
|
||||
? assetService.getInputAssetsIncludingPublic(controller.signal)
|
||||
: Promise.resolve<AssetItem[]>([]),
|
||||
controller
|
||||
),
|
||||
abortSiblingsOnFailure(
|
||||
includeGeneratedAssets
|
||||
? fetchGeneratedAssets(controller.signal, {
|
||||
isCloud,
|
||||
generatedMatchNames,
|
||||
pathOptions
|
||||
})
|
||||
: Promise.resolve<AssetItem[]>([]),
|
||||
controller
|
||||
)
|
||||
])
|
||||
|
||||
return { inputAssets, generatedAssets }
|
||||
} finally {
|
||||
signal?.removeEventListener('abort', abortFromCaller)
|
||||
}
|
||||
}
|
||||
|
||||
interface FetchGeneratedAssetsOptions {
|
||||
isCloud: boolean
|
||||
generatedMatchNames: ReadonlySet<string>
|
||||
pathOptions: MediaPathDetectionOptions
|
||||
}
|
||||
|
||||
export function getAssetDetectionNames(
|
||||
asset: AssetItem,
|
||||
options: MediaPathDetectionOptions
|
||||
): string[] {
|
||||
const names = new Set<string>()
|
||||
// Treat names and hashes as opaque match keys because Cloud may use either in widget values.
|
||||
addPathDetectionNames(names, asset.asset_hash, options)
|
||||
addPathDetectionNames(names, asset.name, options)
|
||||
|
||||
const subfolder = asset.user_metadata?.subfolder
|
||||
if (typeof subfolder === 'string' && subfolder) {
|
||||
addSubfolderPathDetectionNames(names, subfolder, asset.name, options)
|
||||
}
|
||||
|
||||
return Array.from(names)
|
||||
}
|
||||
|
||||
async function fetchGeneratedAssets(
|
||||
signal: AbortSignal | undefined,
|
||||
{ isCloud, generatedMatchNames, pathOptions }: FetchGeneratedAssetsOptions
|
||||
): Promise<AssetItem[]> {
|
||||
if (isCloud) {
|
||||
return await fetchCloudGeneratedAssets(
|
||||
signal,
|
||||
generatedMatchNames,
|
||||
pathOptions
|
||||
)
|
||||
}
|
||||
|
||||
return await fetchGeneratedHistoryAssets(
|
||||
signal,
|
||||
generatedMatchNames,
|
||||
pathOptions
|
||||
)
|
||||
}
|
||||
|
||||
async function fetchCloudGeneratedAssets(
|
||||
signal: AbortSignal | undefined,
|
||||
targetNames: ReadonlySet<string>,
|
||||
pathOptions: MediaPathDetectionOptions
|
||||
): Promise<AssetItem[]> {
|
||||
const assets: AssetItem[] = []
|
||||
const foundTargetNames = new Set<string>()
|
||||
let offset = 0
|
||||
|
||||
while (true) {
|
||||
signal?.throwIfAborted()
|
||||
|
||||
const assetPage = await assetService.getAssetsPageByTag('output', true, {
|
||||
limit: CLOUD_OUTPUT_ASSETS_PAGE_SIZE,
|
||||
offset,
|
||||
signal
|
||||
})
|
||||
|
||||
signal?.throwIfAborted()
|
||||
|
||||
const batch = assetPage.assets
|
||||
if (batch.length === 0) return assets
|
||||
|
||||
for (const asset of batch) {
|
||||
assets.push(asset)
|
||||
rememberResolvedTargetNames(
|
||||
asset,
|
||||
targetNames,
|
||||
foundTargetNames,
|
||||
pathOptions
|
||||
)
|
||||
}
|
||||
|
||||
if (
|
||||
!assetPage.has_more ||
|
||||
hasResolvedAllTargetNames(targetNames, foundTargetNames)
|
||||
) {
|
||||
return assets
|
||||
}
|
||||
|
||||
offset += batch.length
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchGeneratedHistoryAssets(
|
||||
signal: AbortSignal | undefined,
|
||||
targetNames: ReadonlySet<string>,
|
||||
pathOptions: MediaPathDetectionOptions
|
||||
): Promise<AssetItem[]> {
|
||||
const assets: AssetItem[] = []
|
||||
const foundTargetNames = new Set<string>()
|
||||
const seenJobIds = new Set<string>()
|
||||
let offset = 0
|
||||
|
||||
while (true) {
|
||||
signal?.throwIfAborted()
|
||||
|
||||
const requestedOffset = offset
|
||||
const historyPage = await fetchHistoryPage(
|
||||
api.fetchApi.bind(api),
|
||||
HISTORY_MEDIA_ASSETS_PAGE_SIZE,
|
||||
requestedOffset
|
||||
)
|
||||
|
||||
signal?.throwIfAborted()
|
||||
|
||||
let newJobCount = 0
|
||||
for (const job of historyPage.jobs) {
|
||||
if (seenJobIds.has(job.id)) continue
|
||||
seenJobIds.add(job.id)
|
||||
newJobCount += 1
|
||||
|
||||
const asset = mapHistoryJobToAsset(job)
|
||||
if (!asset) continue
|
||||
|
||||
assets.push(asset)
|
||||
rememberResolvedTargetNames(
|
||||
asset,
|
||||
targetNames,
|
||||
foundTargetNames,
|
||||
pathOptions
|
||||
)
|
||||
}
|
||||
|
||||
if (
|
||||
!historyPage.hasMore ||
|
||||
historyPage.jobs.length === 0 ||
|
||||
newJobCount === 0 ||
|
||||
hasResolvedAllTargetNames(targetNames, foundTargetNames)
|
||||
) {
|
||||
return assets
|
||||
}
|
||||
|
||||
offset = requestedOffset + historyPage.jobs.length
|
||||
}
|
||||
}
|
||||
|
||||
async function abortSiblingsOnFailure<T>(
|
||||
promise: Promise<T>,
|
||||
controller: AbortController
|
||||
): Promise<T> {
|
||||
try {
|
||||
return await promise
|
||||
} catch (err) {
|
||||
if (!controller.signal.aborted) controller.abort(err)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
function addPathDetectionNames(
|
||||
names: Set<string>,
|
||||
value: string | null | undefined,
|
||||
options: MediaPathDetectionOptions
|
||||
) {
|
||||
if (!value) return
|
||||
for (const name of getMediaPathDetectionNames(value, options)) {
|
||||
names.add(name)
|
||||
}
|
||||
}
|
||||
|
||||
function addSubfolderPathDetectionNames(
|
||||
names: Set<string>,
|
||||
subfolder: string,
|
||||
value: string | null | undefined,
|
||||
options: MediaPathDetectionOptions
|
||||
) {
|
||||
if (!value) return
|
||||
|
||||
const filePath = joinFilePath(subfolder, value)
|
||||
for (const path of getFilePathSeparatorVariants(filePath)) {
|
||||
addPathDetectionNames(names, path, options)
|
||||
}
|
||||
}
|
||||
|
||||
function rememberResolvedTargetNames(
|
||||
asset: AssetItem,
|
||||
targetNames: ReadonlySet<string>,
|
||||
foundTargetNames: Set<string>,
|
||||
options: MediaPathDetectionOptions
|
||||
) {
|
||||
if (targetNames.size === 0) return
|
||||
|
||||
for (const name of getAssetDetectionNames(asset, options)) {
|
||||
if (targetNames.has(name)) foundTargetNames.add(name)
|
||||
}
|
||||
}
|
||||
|
||||
function hasResolvedAllTargetNames(
|
||||
targetNames: ReadonlySet<string>,
|
||||
foundTargetNames: ReadonlySet<string>
|
||||
): boolean {
|
||||
return targetNames.size > 0 && foundTargetNames.size === targetNames.size
|
||||
}
|
||||
|
||||
function mapHistoryJobToAsset(job: JobListItem): AssetItem | null {
|
||||
const output = job.preview_output
|
||||
if (job.status !== 'completed' || !output?.filename) return null
|
||||
|
||||
return {
|
||||
id: `${job.id}-${output.filename}`,
|
||||
name: output.filename,
|
||||
display_name: output.display_name,
|
||||
mime_type: null,
|
||||
tags: ['output'],
|
||||
user_metadata: {
|
||||
subfolder: output.subfolder
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,17 +6,26 @@ import type { LGraphNode } from '@/lib/litegraph/src/LGraphNode'
|
||||
import type { IComboWidget } from '@/lib/litegraph/src/types/widgets'
|
||||
import type { AssetItem } from '@/platform/assets/schemas/assetSchema'
|
||||
import type * as AssetServiceModule from '@/platform/assets/services/assetService'
|
||||
import type * as FetchJobsModule from '@/platform/remote/comfyui/jobs/fetchJobs'
|
||||
import type { JobListItem } from '@/platform/remote/comfyui/jobs/jobTypes'
|
||||
import type { MissingMediaAssetResolver } from './missingMediaAssetResolver'
|
||||
import {
|
||||
scanAllMediaCandidates,
|
||||
scanNodeMediaCandidates,
|
||||
verifyCloudMediaCandidates,
|
||||
verifyMediaCandidates,
|
||||
groupCandidatesByName,
|
||||
groupCandidatesByMediaType
|
||||
} from './missingMediaScan'
|
||||
import type { MissingMediaCandidate } from './types'
|
||||
|
||||
const { mockGetInputAssetsIncludingPublic } = vi.hoisted(() => ({
|
||||
mockGetInputAssetsIncludingPublic: vi.fn()
|
||||
const { mockGetInputAssetsIncludingPublic, mockGetAssetsPageByTag } =
|
||||
vi.hoisted(() => ({
|
||||
mockGetInputAssetsIncludingPublic: vi.fn(),
|
||||
mockGetAssetsPageByTag: vi.fn()
|
||||
}))
|
||||
|
||||
const { mockFetchHistoryPage } = vi.hoisted(() => ({
|
||||
mockFetchHistoryPage: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('@/utils/graphTraversalUtil', () => ({
|
||||
@@ -36,11 +45,23 @@ vi.mock('@/platform/assets/services/assetService', async () => {
|
||||
...actual,
|
||||
assetService: {
|
||||
...actual.assetService,
|
||||
getInputAssetsIncludingPublic: mockGetInputAssetsIncludingPublic
|
||||
getInputAssetsIncludingPublic: mockGetInputAssetsIncludingPublic,
|
||||
getAssetsPageByTag: mockGetAssetsPageByTag
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@/platform/remote/comfyui/jobs/fetchJobs', async () => {
|
||||
const actual = await vi.importActual<typeof FetchJobsModule>(
|
||||
'@/platform/remote/comfyui/jobs/fetchJobs'
|
||||
)
|
||||
|
||||
return {
|
||||
...actual,
|
||||
fetchHistoryPage: mockFetchHistoryPage
|
||||
}
|
||||
})
|
||||
|
||||
function makeCandidate(
|
||||
nodeId: string,
|
||||
name: string,
|
||||
@@ -100,6 +121,43 @@ function makeAsset(name: string, assetHash: string | null = null): AssetItem {
|
||||
}
|
||||
}
|
||||
|
||||
function makeAssetResolver(
|
||||
inputAssets: AssetItem[],
|
||||
generatedAssets: AssetItem[] = []
|
||||
): MissingMediaAssetResolver {
|
||||
return vi.fn(async () => ({ inputAssets, generatedAssets }))
|
||||
}
|
||||
|
||||
function makeAssetPage(
|
||||
assets: AssetItem[],
|
||||
options: { hasMore?: boolean; total?: number } = {}
|
||||
) {
|
||||
return {
|
||||
assets,
|
||||
total: options.total ?? assets.length,
|
||||
has_more: options.hasMore ?? false
|
||||
}
|
||||
}
|
||||
|
||||
function makeHistoryJob(
|
||||
filename: string,
|
||||
options: { id?: string; subfolder?: string } = {}
|
||||
): JobListItem {
|
||||
return fromAny<JobListItem, unknown>({
|
||||
id: options.id ?? filename,
|
||||
status: 'completed',
|
||||
create_time: 0,
|
||||
priority: 0,
|
||||
preview_output: {
|
||||
filename,
|
||||
subfolder: options.subfolder ?? '',
|
||||
type: 'output',
|
||||
nodeId: '1',
|
||||
mediaType: 'images'
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
describe('scanNodeMediaCandidates', () => {
|
||||
it('returns candidate for a LoadImage node with missing image', () => {
|
||||
const graph = makeGraph([])
|
||||
@@ -145,6 +203,131 @@ describe('scanNodeMediaCandidates', () => {
|
||||
|
||||
expect(result).toEqual([])
|
||||
})
|
||||
|
||||
it.each([
|
||||
{
|
||||
nodeType: 'LoadImage',
|
||||
widgetName: 'image',
|
||||
mediaType: 'image',
|
||||
value: 'photo.png [input]',
|
||||
option: 'photo.png'
|
||||
},
|
||||
{
|
||||
nodeType: 'LoadImageMask',
|
||||
widgetName: 'image',
|
||||
mediaType: 'image',
|
||||
value: 'mask.png [input]',
|
||||
option: 'mask.png'
|
||||
},
|
||||
{
|
||||
nodeType: 'LoadVideo',
|
||||
widgetName: 'file',
|
||||
mediaType: 'video',
|
||||
value: 'clip.mp4 [input]',
|
||||
option: 'clip.mp4'
|
||||
},
|
||||
{
|
||||
nodeType: 'LoadAudio',
|
||||
widgetName: 'audio',
|
||||
mediaType: 'audio',
|
||||
value: 'sound.wav [input]',
|
||||
option: 'sound.wav'
|
||||
}
|
||||
])(
|
||||
'matches annotated $nodeType values against clean OSS options',
|
||||
({ nodeType, widgetName, mediaType, value, option }) => {
|
||||
const graph = makeGraph([])
|
||||
const node = makeMediaNode(
|
||||
1,
|
||||
nodeType,
|
||||
[makeMediaCombo(widgetName, value, [option])],
|
||||
0
|
||||
)
|
||||
|
||||
const result = scanNodeMediaCandidates(graph, node, false)
|
||||
|
||||
expect(result).toHaveLength(1)
|
||||
expect(result[0]).toMatchObject({
|
||||
nodeType,
|
||||
widgetName,
|
||||
mediaType,
|
||||
name: value,
|
||||
isMissing: false
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
it.each([
|
||||
{
|
||||
nodeType: 'LoadImage',
|
||||
widgetName: 'image',
|
||||
value: 'photo.png [output]'
|
||||
},
|
||||
{
|
||||
nodeType: 'LoadVideo',
|
||||
widgetName: 'file',
|
||||
value: 'clip.mp4 [output]'
|
||||
},
|
||||
{
|
||||
nodeType: 'LoadAudio',
|
||||
widgetName: 'audio',
|
||||
value: 'sound.wav [output]'
|
||||
}
|
||||
])(
|
||||
'leaves OSS $nodeType output annotations pending when not in options',
|
||||
({ nodeType, widgetName, value }) => {
|
||||
const graph = makeGraph([])
|
||||
const node = makeMediaNode(
|
||||
1,
|
||||
nodeType,
|
||||
[makeMediaCombo(widgetName, value, ['other-file.png', value])],
|
||||
0
|
||||
)
|
||||
|
||||
const result = scanNodeMediaCandidates(graph, node, false)
|
||||
|
||||
expect(result[0]).toMatchObject({
|
||||
nodeType,
|
||||
widgetName,
|
||||
name: value,
|
||||
isMissing: undefined
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
it('marks OSS input annotations missing when the clean option is absent', () => {
|
||||
const graph = makeGraph([])
|
||||
const node = makeMediaNode(
|
||||
1,
|
||||
'LoadImage',
|
||||
[makeMediaCombo('image', 'photo.png [input]', ['other.png'])],
|
||||
0
|
||||
)
|
||||
|
||||
const result = scanNodeMediaCandidates(graph, node, false)
|
||||
|
||||
expect(result[0]).toMatchObject({
|
||||
name: 'photo.png [input]',
|
||||
isMissing: true
|
||||
})
|
||||
})
|
||||
|
||||
it('does not treat compact Cloud annotations as valid OSS options', () => {
|
||||
const graph = makeGraph([])
|
||||
const node = makeMediaNode(
|
||||
1,
|
||||
'LoadImage',
|
||||
[makeMediaCombo('image', 'photo.png[input]', ['photo.png'])],
|
||||
0
|
||||
)
|
||||
|
||||
const result = scanNodeMediaCandidates(graph, node, false)
|
||||
|
||||
expect(result[0]).toMatchObject({
|
||||
name: 'photo.png[input]',
|
||||
isMissing: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('scanAllMediaCandidates', () => {
|
||||
@@ -261,7 +444,7 @@ describe('groupCandidatesByMediaType', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('verifyCloudMediaCandidates', () => {
|
||||
describe('verifyMediaCandidates', () => {
|
||||
const existingHash =
|
||||
'blake3:1111111111111111111111111111111111111111111111111111111111111111'
|
||||
const missingHash =
|
||||
@@ -270,6 +453,14 @@ describe('verifyCloudMediaCandidates', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockGetInputAssetsIncludingPublic.mockResolvedValue([])
|
||||
mockGetAssetsPageByTag.mockResolvedValue(makeAssetPage([]))
|
||||
mockFetchHistoryPage.mockResolvedValue({
|
||||
jobs: [],
|
||||
total: 0,
|
||||
offset: 0,
|
||||
limit: 200,
|
||||
hasMore: false
|
||||
})
|
||||
})
|
||||
|
||||
it('matches candidates by available input asset name or hash', async () => {
|
||||
@@ -278,16 +469,25 @@ describe('verifyCloudMediaCandidates', () => {
|
||||
makeCandidate('2', existingHash, { isMissing: undefined }),
|
||||
makeCandidate('3', missingHash, { isMissing: undefined })
|
||||
]
|
||||
const fetchInputAssets = vi.fn(async () => [
|
||||
const resolveAssetSources = makeAssetResolver([
|
||||
makeAsset('photo.png', existingHash)
|
||||
])
|
||||
|
||||
await verifyCloudMediaCandidates(candidates, undefined, fetchInputAssets)
|
||||
await verifyMediaCandidates(candidates, {
|
||||
isCloud: true,
|
||||
resolveAssetSources
|
||||
})
|
||||
|
||||
expect(candidates[0].isMissing).toBe(false)
|
||||
expect(candidates[1].isMissing).toBe(false)
|
||||
expect(candidates[2].isMissing).toBe(true)
|
||||
expect(fetchInputAssets).toHaveBeenCalledOnce()
|
||||
expect(resolveAssetSources).toHaveBeenCalledWith({
|
||||
signal: undefined,
|
||||
isCloud: true,
|
||||
includeGeneratedAssets: false,
|
||||
generatedMatchNames: new Set(),
|
||||
allowCompactSuffix: true
|
||||
})
|
||||
})
|
||||
|
||||
it('matches asset names when asset_hash is null', async () => {
|
||||
@@ -295,22 +495,202 @@ describe('verifyCloudMediaCandidates', () => {
|
||||
makeCandidate('1', 'legacy-photo.png', { isMissing: undefined }),
|
||||
makeCandidate('2', 'missing-photo.png', { isMissing: undefined })
|
||||
]
|
||||
const fetchInputAssets = vi.fn(async () => [
|
||||
const resolveAssetSources = makeAssetResolver([
|
||||
makeAsset('legacy-photo.png', null)
|
||||
])
|
||||
|
||||
await verifyCloudMediaCandidates(candidates, undefined, fetchInputAssets)
|
||||
await verifyMediaCandidates(candidates, {
|
||||
isCloud: true,
|
||||
resolveAssetSources
|
||||
})
|
||||
|
||||
expect(candidates[0].isMissing).toBe(false)
|
||||
expect(candidates[1].isMissing).toBe(true)
|
||||
})
|
||||
|
||||
it('matches annotated candidate names against clean asset names', async () => {
|
||||
const candidates = [
|
||||
makeCandidate('1', 'photo.png [input]', { isMissing: undefined }),
|
||||
makeCandidate('2', 'clip.mp4[input]', {
|
||||
nodeType: 'LoadVideo',
|
||||
widgetName: 'file',
|
||||
mediaType: 'video',
|
||||
isMissing: undefined
|
||||
}),
|
||||
makeCandidate('3', 'missing.wav [output]', {
|
||||
nodeType: 'LoadAudio',
|
||||
widgetName: 'audio',
|
||||
mediaType: 'audio',
|
||||
isMissing: undefined
|
||||
})
|
||||
]
|
||||
const resolveAssetSources = makeAssetResolver(
|
||||
[makeAsset('photo.png'), makeAsset('clip.mp4')],
|
||||
[]
|
||||
)
|
||||
|
||||
await verifyMediaCandidates(candidates, {
|
||||
isCloud: true,
|
||||
resolveAssetSources
|
||||
})
|
||||
|
||||
expect(candidates[0]).toMatchObject({
|
||||
name: 'photo.png [input]',
|
||||
isMissing: false
|
||||
})
|
||||
expect(candidates[1]).toMatchObject({
|
||||
name: 'clip.mp4[input]',
|
||||
isMissing: false
|
||||
})
|
||||
expect(candidates[2]).toMatchObject({
|
||||
name: 'missing.wav [output]',
|
||||
isMissing: true
|
||||
})
|
||||
})
|
||||
|
||||
it('matches output hash filenames against generated media assets', async () => {
|
||||
const candidates = [
|
||||
makeCandidate(
|
||||
'1',
|
||||
'147257c95a3e957e0deee73a077cfec89da2d906dd086ca70a2b0c897a9591d6e.png [output]',
|
||||
{
|
||||
isMissing: undefined
|
||||
}
|
||||
)
|
||||
]
|
||||
const resolveAssetSources = makeAssetResolver(
|
||||
[],
|
||||
[
|
||||
makeAsset(
|
||||
'147257c95a3e957e0deee73a077cfec89da2d906dd086ca70a2b0c897a9591d6e.png'
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
await verifyMediaCandidates(candidates, {
|
||||
isCloud: true,
|
||||
resolveAssetSources
|
||||
})
|
||||
|
||||
expect(resolveAssetSources).toHaveBeenCalledWith({
|
||||
signal: undefined,
|
||||
isCloud: true,
|
||||
includeGeneratedAssets: true,
|
||||
generatedMatchNames: new Set([
|
||||
'147257c95a3e957e0deee73a077cfec89da2d906dd086ca70a2b0c897a9591d6e.png'
|
||||
]),
|
||||
allowCompactSuffix: true
|
||||
})
|
||||
expect(candidates[0]).toMatchObject({
|
||||
name: '147257c95a3e957e0deee73a077cfec89da2d906dd086ca70a2b0c897a9591d6e.png [output]',
|
||||
isMissing: false
|
||||
})
|
||||
})
|
||||
|
||||
it('does not satisfy output annotations with input assets of the same name', async () => {
|
||||
const candidates = [
|
||||
makeCandidate('1', 'photo.png [output]', { isMissing: undefined })
|
||||
]
|
||||
const resolveAssetSources = makeAssetResolver([makeAsset('photo.png')])
|
||||
|
||||
await verifyMediaCandidates(candidates, {
|
||||
isCloud: true,
|
||||
resolveAssetSources
|
||||
})
|
||||
|
||||
expect(candidates[0].isMissing).toBe(true)
|
||||
})
|
||||
|
||||
it('does not satisfy input candidates with output assets of the same name', async () => {
|
||||
const candidates = [
|
||||
makeCandidate('1', 'photo.png', { isMissing: undefined })
|
||||
]
|
||||
const resolveAssetSources = makeAssetResolver([], [makeAsset('photo.png')])
|
||||
|
||||
await verifyMediaCandidates(candidates, {
|
||||
isCloud: true,
|
||||
resolveAssetSources
|
||||
})
|
||||
|
||||
expect(candidates[0].isMissing).toBe(true)
|
||||
})
|
||||
|
||||
it('verifies OSS output candidates against generated history without cloud assets', async () => {
|
||||
const candidates = [
|
||||
makeCandidate('1', 'subfolder/photo.png [output]', {
|
||||
isMissing: undefined
|
||||
})
|
||||
]
|
||||
|
||||
mockFetchHistoryPage.mockResolvedValueOnce({
|
||||
jobs: [makeHistoryJob('photo.png', { subfolder: 'subfolder' })],
|
||||
total: 1,
|
||||
offset: 0,
|
||||
limit: 200,
|
||||
hasMore: false
|
||||
})
|
||||
|
||||
await verifyMediaCandidates(candidates, { isCloud: false })
|
||||
|
||||
expect(mockGetInputAssetsIncludingPublic).not.toHaveBeenCalled()
|
||||
expect(mockFetchHistoryPage).toHaveBeenCalledWith(
|
||||
expect.any(Function),
|
||||
200,
|
||||
0
|
||||
)
|
||||
expect(candidates[0]).toMatchObject({
|
||||
name: 'subfolder/photo.png [output]',
|
||||
isMissing: false
|
||||
})
|
||||
})
|
||||
|
||||
it('does not normalize compact annotations when verifying OSS candidates', async () => {
|
||||
const candidates = [
|
||||
makeCandidate('1', 'photo.png[output]', { isMissing: undefined })
|
||||
]
|
||||
const resolveAssetSources = makeAssetResolver([makeAsset('photo.png')])
|
||||
|
||||
await verifyMediaCandidates(candidates, {
|
||||
isCloud: false,
|
||||
resolveAssetSources
|
||||
})
|
||||
|
||||
expect(resolveAssetSources).toHaveBeenCalledWith({
|
||||
signal: undefined,
|
||||
isCloud: false,
|
||||
includeGeneratedAssets: false,
|
||||
generatedMatchNames: new Set(),
|
||||
allowCompactSuffix: false
|
||||
})
|
||||
expect(candidates[0].isMissing).toBe(true)
|
||||
})
|
||||
|
||||
it('matches when the asset identifier itself is annotated', async () => {
|
||||
const candidates = [
|
||||
makeCandidate('1', 'clip.mp4[output]', { isMissing: undefined })
|
||||
]
|
||||
const resolveAssetSources = makeAssetResolver(
|
||||
[],
|
||||
[makeAsset('clip.mp4 [output]')]
|
||||
)
|
||||
|
||||
await verifyMediaCandidates(candidates, {
|
||||
isCloud: true,
|
||||
resolveAssetSources
|
||||
})
|
||||
|
||||
expect(candidates[0].isMissing).toBe(false)
|
||||
})
|
||||
|
||||
it('marks pending candidates missing when no input assets are available', async () => {
|
||||
const candidates = [
|
||||
makeCandidate('1', 'photo.png', { isMissing: undefined })
|
||||
]
|
||||
|
||||
await verifyCloudMediaCandidates(candidates, undefined, async () => [])
|
||||
await verifyMediaCandidates(candidates, {
|
||||
isCloud: true,
|
||||
resolveAssetSources: makeAssetResolver([])
|
||||
})
|
||||
|
||||
expect(candidates[0].isMissing).toBe(true)
|
||||
})
|
||||
@@ -323,10 +703,104 @@ describe('verifyCloudMediaCandidates', () => {
|
||||
makeAsset('stored-photo.png', existingHash)
|
||||
])
|
||||
|
||||
await verifyCloudMediaCandidates(candidates)
|
||||
await verifyMediaCandidates(candidates, { isCloud: true })
|
||||
|
||||
expect(candidates[0].isMissing).toBe(false)
|
||||
expect(mockGetInputAssetsIncludingPublic).toHaveBeenCalledWith(undefined)
|
||||
expect(mockGetInputAssetsIncludingPublic).toHaveBeenCalledWith(
|
||||
expect.any(AbortSignal)
|
||||
)
|
||||
expect(mockFetchHistoryPage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reads cloud output assets by tag for output candidates', async () => {
|
||||
const outputHash =
|
||||
'147257c95a3e957e0deee73a077cfec89da2d906dd086ca70a2b0c897a9591d6e.png'
|
||||
const candidates = [
|
||||
makeCandidate('1', `${outputHash} [output]`, { isMissing: undefined })
|
||||
]
|
||||
mockGetAssetsPageByTag.mockResolvedValue(
|
||||
makeAssetPage([makeAsset(outputHash)])
|
||||
)
|
||||
|
||||
await verifyMediaCandidates(candidates, { isCloud: true })
|
||||
|
||||
expect(mockGetInputAssetsIncludingPublic).toHaveBeenCalledWith(
|
||||
expect.any(AbortSignal)
|
||||
)
|
||||
expect(mockGetAssetsPageByTag).toHaveBeenCalledWith(
|
||||
'output',
|
||||
true,
|
||||
expect.objectContaining({
|
||||
limit: 500,
|
||||
offset: 0,
|
||||
signal: expect.any(AbortSignal)
|
||||
})
|
||||
)
|
||||
expect(mockFetchHistoryPage).not.toHaveBeenCalled()
|
||||
expect(candidates[0].isMissing).toBe(false)
|
||||
})
|
||||
|
||||
it('walks OSS generated history pages until hasMore is false', async () => {
|
||||
const outputHash =
|
||||
'147257c95a3e957e0deee73a077cfec89da2d906dd086ca70a2b0c897a9591d6e.png'
|
||||
const candidates = [
|
||||
makeCandidate('1', `${outputHash} [output]`, { isMissing: undefined })
|
||||
]
|
||||
mockFetchHistoryPage
|
||||
.mockResolvedValueOnce({
|
||||
jobs: Array.from({ length: 200 }, (_, index) =>
|
||||
makeHistoryJob(`other-${index}.png`)
|
||||
),
|
||||
total: 201,
|
||||
offset: 0,
|
||||
limit: 200,
|
||||
hasMore: true
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
jobs: [makeHistoryJob(outputHash)],
|
||||
total: 201,
|
||||
offset: 200,
|
||||
limit: 200,
|
||||
hasMore: false
|
||||
})
|
||||
|
||||
await verifyMediaCandidates(candidates, { isCloud: false })
|
||||
|
||||
expect(mockFetchHistoryPage).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.any(Function),
|
||||
200,
|
||||
0
|
||||
)
|
||||
expect(mockFetchHistoryPage).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.any(Function),
|
||||
200,
|
||||
200
|
||||
)
|
||||
expect(candidates[0].isMissing).toBe(false)
|
||||
})
|
||||
|
||||
it('trusts OSS history hasMore instead of page length', async () => {
|
||||
const candidates = [
|
||||
makeCandidate('1', 'missing-output.png [output]', {
|
||||
isMissing: undefined
|
||||
})
|
||||
]
|
||||
mockFetchHistoryPage.mockResolvedValueOnce({
|
||||
jobs: Array.from({ length: 200 }, (_, index) =>
|
||||
makeHistoryJob(`other-${index}.png`)
|
||||
),
|
||||
total: 200,
|
||||
offset: 0,
|
||||
limit: 200,
|
||||
hasMore: false
|
||||
})
|
||||
|
||||
await verifyMediaCandidates(candidates, { isCloud: false })
|
||||
|
||||
expect(mockFetchHistoryPage).toHaveBeenCalledOnce()
|
||||
expect(candidates[0].isMissing).toBe(true)
|
||||
})
|
||||
|
||||
it('respects abort signal before execution', async () => {
|
||||
@@ -337,7 +811,10 @@ describe('verifyCloudMediaCandidates', () => {
|
||||
makeCandidate('1', missingHash, { isMissing: undefined })
|
||||
]
|
||||
|
||||
await verifyCloudMediaCandidates(candidates, controller.signal)
|
||||
await verifyMediaCandidates(candidates, {
|
||||
isCloud: true,
|
||||
signal: controller.signal
|
||||
})
|
||||
|
||||
expect(candidates[0].isMissing).toBeUndefined()
|
||||
expect(mockGetInputAssetsIncludingPublic).not.toHaveBeenCalled()
|
||||
@@ -348,16 +825,19 @@ describe('verifyCloudMediaCandidates', () => {
|
||||
const candidates = [
|
||||
makeCandidate('1', existingHash, { isMissing: undefined })
|
||||
]
|
||||
const fetchInputAssets = vi.fn(async () => {
|
||||
const resolveAssetSources: MissingMediaAssetResolver = vi.fn(async () => {
|
||||
controller.abort()
|
||||
return [makeAsset('stored-photo.png', existingHash)]
|
||||
return {
|
||||
inputAssets: [makeAsset('stored-photo.png', existingHash)],
|
||||
generatedAssets: []
|
||||
}
|
||||
})
|
||||
|
||||
await verifyCloudMediaCandidates(
|
||||
candidates,
|
||||
controller.signal,
|
||||
fetchInputAssets
|
||||
)
|
||||
await verifyMediaCandidates(candidates, {
|
||||
isCloud: true,
|
||||
signal: controller.signal,
|
||||
resolveAssetSources
|
||||
})
|
||||
|
||||
expect(candidates[0].isMissing).toBeUndefined()
|
||||
})
|
||||
@@ -365,7 +845,7 @@ describe('verifyCloudMediaCandidates', () => {
|
||||
it('skips candidates already resolved as true', async () => {
|
||||
const candidates = [makeCandidate('1', missingHash, { isMissing: true })]
|
||||
|
||||
await verifyCloudMediaCandidates(candidates)
|
||||
await verifyMediaCandidates(candidates, { isCloud: true })
|
||||
|
||||
expect(candidates[0].isMissing).toBe(true)
|
||||
expect(mockGetInputAssetsIncludingPublic).not.toHaveBeenCalled()
|
||||
@@ -374,7 +854,7 @@ describe('verifyCloudMediaCandidates', () => {
|
||||
it('skips candidates already resolved as false', async () => {
|
||||
const candidates = [makeCandidate('1', existingHash, { isMissing: false })]
|
||||
|
||||
await verifyCloudMediaCandidates(candidates)
|
||||
await verifyMediaCandidates(candidates, { isCloud: true })
|
||||
|
||||
expect(candidates[0].isMissing).toBe(false)
|
||||
expect(mockGetInputAssetsIncludingPublic).not.toHaveBeenCalled()
|
||||
@@ -383,7 +863,7 @@ describe('verifyCloudMediaCandidates', () => {
|
||||
it('skips entirely when no pending candidates', async () => {
|
||||
const candidates = [makeCandidate('1', missingHash, { isMissing: true })]
|
||||
|
||||
await verifyCloudMediaCandidates(candidates)
|
||||
await verifyMediaCandidates(candidates, { isCloud: true })
|
||||
|
||||
expect(mockGetInputAssetsIncludingPublic).not.toHaveBeenCalled()
|
||||
})
|
||||
@@ -398,9 +878,11 @@ describe('verifyCloudMediaCandidates', () => {
|
||||
inputAssets[42] = makeAsset('public-asset-record', 'public-photo.png')
|
||||
mockGetInputAssetsIncludingPublic.mockResolvedValue(inputAssets)
|
||||
|
||||
await verifyCloudMediaCandidates(candidates)
|
||||
await verifyMediaCandidates(candidates, { isCloud: true })
|
||||
|
||||
expect(mockGetInputAssetsIncludingPublic).toHaveBeenCalledWith(undefined)
|
||||
expect(mockGetInputAssetsIncludingPublic).toHaveBeenCalledWith(
|
||||
expect.any(AbortSignal)
|
||||
)
|
||||
expect(candidates[0].isMissing).toBe(false)
|
||||
})
|
||||
|
||||
@@ -411,17 +893,17 @@ describe('verifyCloudMediaCandidates', () => {
|
||||
const candidates = [
|
||||
makeCandidate('1', 'photo.png', { isMissing: undefined })
|
||||
]
|
||||
const fetchInputAssets = vi.fn(async () => {
|
||||
const resolveAssetSources: MissingMediaAssetResolver = vi.fn(async () => {
|
||||
controller.abort()
|
||||
throw abortError
|
||||
})
|
||||
|
||||
await expect(
|
||||
verifyCloudMediaCandidates(
|
||||
candidates,
|
||||
controller.signal,
|
||||
fetchInputAssets
|
||||
)
|
||||
verifyMediaCandidates(candidates, {
|
||||
isCloud: true,
|
||||
signal: controller.signal,
|
||||
resolveAssetSources
|
||||
})
|
||||
).resolves.toBeUndefined()
|
||||
|
||||
expect(candidates[0].isMissing).toBeUndefined()
|
||||
@@ -434,18 +916,24 @@ describe('verifyCloudMediaCandidates', () => {
|
||||
const candidates = [
|
||||
makeCandidate('1', 'photo.png', { isMissing: undefined })
|
||||
]
|
||||
mockGetInputAssetsIncludingPublic.mockImplementationOnce(async () => {
|
||||
controller.abort()
|
||||
throw abortError
|
||||
})
|
||||
let serviceSignal: AbortSignal | undefined
|
||||
mockGetInputAssetsIncludingPublic.mockImplementationOnce(
|
||||
async (signal?: AbortSignal) => {
|
||||
serviceSignal = signal
|
||||
controller.abort()
|
||||
throw abortError
|
||||
}
|
||||
)
|
||||
|
||||
await expect(
|
||||
verifyCloudMediaCandidates(candidates, controller.signal)
|
||||
verifyMediaCandidates(candidates, {
|
||||
isCloud: true,
|
||||
signal: controller.signal
|
||||
})
|
||||
).resolves.toBeUndefined()
|
||||
|
||||
expect(mockGetInputAssetsIncludingPublic).toHaveBeenCalledWith(
|
||||
controller.signal
|
||||
)
|
||||
expect(serviceSignal).toBeInstanceOf(AbortSignal)
|
||||
expect(serviceSignal?.aborted).toBe(true)
|
||||
expect(candidates[0].isMissing).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -19,8 +19,17 @@ import {
|
||||
import { LGraphEventMode } from '@/lib/litegraph/src/types/globalEnums'
|
||||
import { resolveComboValues } from '@/utils/litegraphUtil'
|
||||
import type { AssetItem } from '@/platform/assets/schemas/assetSchema'
|
||||
import { assetService } from '@/platform/assets/services/assetService'
|
||||
import { isAbortError } from '@/utils/typeGuardUtil'
|
||||
import {
|
||||
getAnnotatedMediaPathTypeForDetection,
|
||||
getMediaPathDetectionNames,
|
||||
normalizeAnnotatedMediaPathForDetection
|
||||
} from './mediaPathDetectionUtil'
|
||||
import {
|
||||
getAssetDetectionNames,
|
||||
resolveMissingMediaAssetSources
|
||||
} from './missingMediaAssetResolver'
|
||||
import type { MissingMediaAssetResolver } from './missingMediaAssetResolver'
|
||||
|
||||
/** Map of node types to their media widget name and media type. */
|
||||
const MEDIA_NODE_WIDGETS: Record<
|
||||
@@ -28,6 +37,7 @@ const MEDIA_NODE_WIDGETS: Record<
|
||||
{ widgetName: string; mediaType: MediaType }
|
||||
> = {
|
||||
LoadImage: { widgetName: 'image', mediaType: 'image' },
|
||||
LoadImageMask: { widgetName: 'image', mediaType: 'image' },
|
||||
LoadVideo: { widgetName: 'file', mediaType: 'video' },
|
||||
LoadAudio: { widgetName: 'audio', mediaType: 'audio' }
|
||||
}
|
||||
@@ -39,7 +49,8 @@ function isComboWidget(widget: IBaseWidget): widget is IComboWidget {
|
||||
/**
|
||||
* Scan combo widgets on media nodes for file values that may be missing.
|
||||
*
|
||||
* OSS: `isMissing` resolved immediately via widget options.
|
||||
* OSS: `isMissing` is resolved immediately via widget options unless an
|
||||
* output annotation needs generated-history verification.
|
||||
* Cloud: `isMissing` left `undefined` for async verification.
|
||||
*/
|
||||
export function scanAllMediaCandidates(
|
||||
@@ -92,8 +103,17 @@ export function scanNodeMediaCandidates(
|
||||
if (isCloud) {
|
||||
isMissing = undefined
|
||||
} else {
|
||||
const options = resolveComboValues(widget)
|
||||
isMissing = !options.includes(value)
|
||||
const type = getAnnotatedMediaPathTypeForDetection(value)
|
||||
if (type === 'output') {
|
||||
isMissing = undefined
|
||||
} else {
|
||||
const options = resolveComboValues(widget)
|
||||
const detectionNames = getMediaPathDetectionNames(value)
|
||||
const existsInOptions = detectionNames.some((name) =>
|
||||
options.includes(name)
|
||||
)
|
||||
isMissing = !existsInOptions
|
||||
}
|
||||
}
|
||||
|
||||
candidates.push({
|
||||
@@ -109,29 +129,57 @@ export function scanNodeMediaCandidates(
|
||||
return candidates
|
||||
}
|
||||
|
||||
type InputAssetFetcher = (signal?: AbortSignal) => Promise<AssetItem[]>
|
||||
interface MediaVerificationOptions {
|
||||
isCloud: boolean
|
||||
signal?: AbortSignal
|
||||
resolveAssetSources?: MissingMediaAssetResolver
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify cloud media candidates against input assets available to the user,
|
||||
* including public assets returned by the asset list API.
|
||||
* Verify media candidates against assets available to the current runtime.
|
||||
*
|
||||
* A candidate's `name` may be either a filename or an opaque asset hash.
|
||||
* Cloud-side `asset_hash` is not guaranteed to follow a single shape, so we
|
||||
* match against the union of `asset.name` and `asset.asset_hash`.
|
||||
* match against the union of `asset.name` and `asset.asset_hash`. Output
|
||||
* candidates are matched against Cloud output assets or Core generated-history
|
||||
* assets because Core resolves those annotations against output folders, not
|
||||
* input files.
|
||||
* Cloud accepts compact annotated media paths, so only Cloud verification
|
||||
* normalizes compact suffixes.
|
||||
*/
|
||||
export async function verifyCloudMediaCandidates(
|
||||
export async function verifyMediaCandidates(
|
||||
candidates: MissingMediaCandidate[],
|
||||
signal?: AbortSignal,
|
||||
fetchInputAssets: InputAssetFetcher = fetchMissingInputAssets
|
||||
{
|
||||
isCloud,
|
||||
signal,
|
||||
resolveAssetSources = resolveMissingMediaAssetSources
|
||||
}: MediaVerificationOptions
|
||||
): Promise<void> {
|
||||
if (signal?.aborted) return
|
||||
|
||||
const pending = candidates.filter((c) => c.isMissing === undefined)
|
||||
if (pending.length === 0) return
|
||||
|
||||
// Core stores spaced annotations such as `file.png [output]`; Cloud also
|
||||
// accepts compact forms such as `file.png[output]`.
|
||||
const pathOptions = { allowCompactSuffix: isCloud }
|
||||
const generatedMatchNames = getGeneratedCandidateMatchNames(
|
||||
pending,
|
||||
pathOptions
|
||||
)
|
||||
|
||||
let inputAssets: AssetItem[]
|
||||
let generatedAssets: AssetItem[]
|
||||
try {
|
||||
inputAssets = await fetchInputAssets(signal)
|
||||
const assetSources = await resolveAssetSources({
|
||||
signal,
|
||||
isCloud,
|
||||
includeGeneratedAssets: generatedMatchNames.size > 0,
|
||||
generatedMatchNames,
|
||||
allowCompactSuffix: isCloud
|
||||
})
|
||||
inputAssets = assetSources.inputAssets
|
||||
generatedAssets = assetSources.generatedAssets
|
||||
} catch (err) {
|
||||
if (signal?.aborted || isAbortError(err)) return
|
||||
throw err
|
||||
@@ -139,21 +187,62 @@ export async function verifyCloudMediaCandidates(
|
||||
|
||||
if (signal?.aborted) return
|
||||
|
||||
const assetIdentifiers = new Set<string>()
|
||||
for (const asset of inputAssets) {
|
||||
if (asset.asset_hash) assetIdentifiers.add(asset.asset_hash)
|
||||
if (asset.name) assetIdentifiers.add(asset.name)
|
||||
}
|
||||
const inputAssetIdentifiers = new Set<string>()
|
||||
const outputAssetIdentifiers = new Set<string>()
|
||||
addAssetIdentifiers(inputAssetIdentifiers, inputAssets, pathOptions)
|
||||
addAssetIdentifiers(outputAssetIdentifiers, generatedAssets, pathOptions)
|
||||
|
||||
for (const candidate of pending) {
|
||||
candidate.isMissing = !assetIdentifiers.has(candidate.name)
|
||||
const detectionNames = getMediaPathDetectionNames(
|
||||
candidate.name,
|
||||
pathOptions
|
||||
)
|
||||
const type = getAnnotatedMediaPathTypeForDetection(
|
||||
candidate.name,
|
||||
pathOptions
|
||||
)
|
||||
const identifiers =
|
||||
type === 'output' ? outputAssetIdentifiers : inputAssetIdentifiers
|
||||
candidate.isMissing = !detectionNames.some((name) => identifiers.has(name))
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchMissingInputAssets(
|
||||
signal?: AbortSignal
|
||||
): Promise<AssetItem[]> {
|
||||
return await assetService.getInputAssetsIncludingPublic(signal)
|
||||
function getGeneratedCandidateMatchNames(
|
||||
candidates: MissingMediaCandidate[],
|
||||
pathOptions: { allowCompactSuffix: boolean }
|
||||
): Set<string> {
|
||||
const names = new Set<string>()
|
||||
for (const candidate of candidates) {
|
||||
if (!isGeneratedCandidate(candidate, pathOptions)) continue
|
||||
|
||||
names.add(
|
||||
normalizeAnnotatedMediaPathForDetection(candidate.name, pathOptions)
|
||||
)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
function isGeneratedCandidate(
|
||||
candidate: MissingMediaCandidate,
|
||||
pathOptions: { allowCompactSuffix: boolean }
|
||||
): boolean {
|
||||
const type = getAnnotatedMediaPathTypeForDetection(
|
||||
candidate.name,
|
||||
pathOptions
|
||||
)
|
||||
return type === 'output'
|
||||
}
|
||||
|
||||
function addAssetIdentifiers(
|
||||
identifiers: Set<string>,
|
||||
assets: AssetItem[],
|
||||
pathOptions: { allowCompactSuffix: boolean }
|
||||
) {
|
||||
for (const asset of assets) {
|
||||
for (const name of getAssetDetectionNames(asset, pathOptions)) {
|
||||
identifiers.add(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Group confirmed-missing candidates by file name into view models. */
|
||||
|
||||
@@ -16,7 +16,9 @@ export interface MissingMediaCandidate {
|
||||
/**
|
||||
* - `true` — confirmed missing
|
||||
* - `false` — confirmed present
|
||||
* - `undefined` — pending async verification (cloud only)
|
||||
* - `undefined` — pending async verification. Cloud candidates start pending;
|
||||
* OSS output annotated paths may also be deferred to generated-history
|
||||
* verification.
|
||||
*/
|
||||
isMissing: boolean | undefined
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
extractWorkflow,
|
||||
fetchHistory,
|
||||
fetchHistoryPage,
|
||||
fetchJobDetail,
|
||||
fetchQueue
|
||||
} from '@/platform/remote/comfyui/jobs/fetchJobs'
|
||||
@@ -29,15 +30,16 @@ function createMockJob(
|
||||
|
||||
function createMockResponse(
|
||||
jobs: RawJobListItem[],
|
||||
total: number = jobs.length
|
||||
total: number = jobs.length,
|
||||
pagination: Partial<JobsListResponse['pagination']> = {}
|
||||
): JobsListResponse {
|
||||
return {
|
||||
jobs,
|
||||
pagination: {
|
||||
offset: 0,
|
||||
limit: 200,
|
||||
offset: pagination.offset ?? 0,
|
||||
limit: pagination.limit ?? 200,
|
||||
total,
|
||||
has_more: false
|
||||
has_more: pagination.has_more ?? false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -100,7 +102,8 @@ describe('fetchJobs', () => {
|
||||
createMockJob('job4', 'completed'),
|
||||
createMockJob('job5', 'completed')
|
||||
],
|
||||
10 // total of 10 jobs
|
||||
10, // total of 10 jobs
|
||||
{ offset: 5 }
|
||||
)
|
||||
)
|
||||
})
|
||||
@@ -185,6 +188,36 @@ describe('fetchJobs', () => {
|
||||
expect(result[1].id).toBe('text-job')
|
||||
expect(result[2].id).toBe('no-preview-job')
|
||||
})
|
||||
|
||||
it('returns server pagination metadata for history pages', async () => {
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () =>
|
||||
Promise.resolve(
|
||||
createMockResponse(
|
||||
[
|
||||
createMockJob('job4', 'completed'),
|
||||
createMockJob('job5', 'completed')
|
||||
],
|
||||
10,
|
||||
{ offset: 5, limit: 2, has_more: true }
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
const result = await fetchHistoryPage(mockFetch, 2, 5)
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'/jobs?status=completed,failed,cancelled&limit=2&offset=5'
|
||||
)
|
||||
expect(result.jobs).toHaveLength(2)
|
||||
expect(result.offset).toBe(5)
|
||||
expect(result.limit).toBe(2)
|
||||
expect(result.total).toBe(10)
|
||||
expect(result.hasMore).toBe(true)
|
||||
expect(result.jobs[0].priority).toBe(5)
|
||||
expect(result.jobs[1].priority).toBe(4)
|
||||
})
|
||||
})
|
||||
|
||||
describe('fetchQueue', () => {
|
||||
|
||||
@@ -22,6 +22,16 @@ interface FetchJobsRawResult {
|
||||
jobs: RawJobListItem[]
|
||||
total: number
|
||||
offset: number
|
||||
limit: number
|
||||
hasMore: boolean
|
||||
}
|
||||
|
||||
export interface FetchHistoryPageResult {
|
||||
jobs: JobListItem[]
|
||||
total: number
|
||||
offset: number
|
||||
limit: number
|
||||
hasMore: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -40,13 +50,25 @@ async function fetchJobsRaw(
|
||||
const res = await fetchApi(url)
|
||||
if (!res.ok) {
|
||||
console.error(`[Jobs API] Failed to fetch jobs: ${res.status}`)
|
||||
return { jobs: [], total: 0, offset: 0 }
|
||||
return {
|
||||
jobs: [],
|
||||
total: 0,
|
||||
offset,
|
||||
limit: maxItems,
|
||||
hasMore: false
|
||||
}
|
||||
}
|
||||
const data = zJobsListResponse.parse(await res.json())
|
||||
return { jobs: data.jobs, total: data.pagination.total, offset }
|
||||
return {
|
||||
jobs: data.jobs,
|
||||
total: data.pagination.total,
|
||||
offset: data.pagination.offset,
|
||||
limit: data.pagination.limit,
|
||||
hasMore: data.pagination.has_more
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[Jobs API] Error fetching jobs:', error)
|
||||
return { jobs: [], total: 0, offset: 0 }
|
||||
return { jobs: [], total: 0, offset, limit: maxItems, hasMore: false }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,14 +98,33 @@ export async function fetchHistory(
|
||||
maxItems: number = 200,
|
||||
offset: number = 0
|
||||
): Promise<JobListItem[]> {
|
||||
const { jobs, total } = await fetchJobsRaw(
|
||||
const { jobs } = await fetchHistoryPage(fetchApi, maxItems, offset)
|
||||
return jobs
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches one page of history with server-provided pagination metadata.
|
||||
*/
|
||||
export async function fetchHistoryPage(
|
||||
fetchApi: (url: string) => Promise<Response>,
|
||||
maxItems: number = 200,
|
||||
offset: number = 0
|
||||
): Promise<FetchHistoryPageResult> {
|
||||
const result = await fetchJobsRaw(
|
||||
fetchApi,
|
||||
['completed', 'failed', 'cancelled'],
|
||||
maxItems,
|
||||
offset
|
||||
)
|
||||
|
||||
// History gets priority based on total count (lower than queue)
|
||||
return assignPriority(jobs, total - offset)
|
||||
return {
|
||||
jobs: assignPriority(result.jobs, result.total - result.offset),
|
||||
total: result.total,
|
||||
offset: result.offset,
|
||||
limit: result.limit,
|
||||
hasMore: result.hasMore
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -96,7 +96,7 @@ import { useMissingMediaStore } from '@/platform/missingMedia/missingMediaStore'
|
||||
import type { MissingMediaCandidate } from '@/platform/missingMedia/types'
|
||||
import {
|
||||
scanAllMediaCandidates,
|
||||
verifyCloudMediaCandidates
|
||||
verifyMediaCandidates
|
||||
} from '@/platform/missingMedia/missingMediaScan'
|
||||
|
||||
import { anyItemOverlapsRect } from '@/utils/mathUtil'
|
||||
@@ -1508,9 +1508,13 @@ export class ComfyApp {
|
||||
return
|
||||
}
|
||||
|
||||
if (isCloud) {
|
||||
const pending = candidates.some((c) => c.isMissing === undefined)
|
||||
if (pending) {
|
||||
const controller = missingMediaStore.createVerificationAbortController()
|
||||
void verifyCloudMediaCandidates(candidates, controller.signal)
|
||||
void verifyMediaCandidates(candidates, {
|
||||
isCloud,
|
||||
signal: controller.signal
|
||||
})
|
||||
.then(() => {
|
||||
if (controller.signal.aborted) return
|
||||
// Re-check ancestor after async verification (see model pipeline).
|
||||
|
||||
Reference in New Issue
Block a user