mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-04-29 18:52:19 +00:00
[feat] Add model metadata fetching with loading skeleton and gated repo support (#9415)
## Summary - Fetch file sizes via HEAD requests (HuggingFace) and Civitai API with caching and request deduplication - Show skeleton loader while metadata is loading - Display "Accept terms" link for gated HuggingFace models instead of download button ## Changes - **`missingModelsUtils.ts`**: Add `fetchModelMetadata` with Civitai API support, HuggingFace gated repo detection, in-memory cache, and inflight request deduplication - **`MissingModelsContent.vue`**: Add Skeleton loading state, gated model "Accept terms" link, extract `showSkeleton` helper - **`missingModelsUtils.test.ts`**: Tests for HEAD/Civitai fetching, gated repo detection, caching, and deduplication - **`main.json`**: Add `acceptTerms` i18n key ## Related Issues https://github.com/Comfy-Org/ComfyUI_frontend/issues/9410 https://github.com/Comfy-Org/ComfyUI_frontend/issues/9412 ┆Issue is synchronized with this [Notion page](https://www.notion.so/PR-9415-feat-Add-model-metadata-fetching-with-loading-skeleton-and-gated-repo-support-31a6d73d36508127859efa0b3847505e) by [Unito](https://www.unito.io)
This commit is contained in:
@@ -29,14 +29,24 @@
|
|||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="flex shrink-0 items-center gap-2">
|
<div class="flex shrink-0 items-center gap-2">
|
||||||
|
<Skeleton v-if="showSkeleton(model)" class="ml-1.5 h-4 w-12" />
|
||||||
<span
|
<span
|
||||||
v-if="model.isDownloadable && fileSizes.get(model.url)"
|
v-else-if="model.isDownloadable && fileSizes.get(model.url)"
|
||||||
class="text-xs text-muted-foreground"
|
class="pl-1.5 text-xs text-muted-foreground"
|
||||||
>
|
>
|
||||||
{{ formatSize(fileSizes.get(model.url)) }}
|
{{ formatSize(fileSizes.get(model.url)) }}
|
||||||
</span>
|
</span>
|
||||||
|
<a
|
||||||
|
v-else-if="gatedModelUrls.has(model.url)"
|
||||||
|
:href="gatedModelUrls.get(model.url)"
|
||||||
|
target="_blank"
|
||||||
|
rel="noopener noreferrer"
|
||||||
|
class="text-xs text-primary hover:underline"
|
||||||
|
>
|
||||||
|
{{ $t('missingModelsDialog.acceptTerms') }}
|
||||||
|
</a>
|
||||||
<Button
|
<Button
|
||||||
v-if="model.isDownloadable"
|
v-else-if="model.isDownloadable"
|
||||||
variant="textonly"
|
variant="textonly"
|
||||||
size="icon"
|
size="icon"
|
||||||
:title="model.url"
|
:title="model.url"
|
||||||
@@ -100,15 +110,17 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, onMounted, reactive } from 'vue'
|
import { computed, onMounted, reactive, ref } from 'vue'
|
||||||
|
|
||||||
import Button from '@/components/ui/button/Button.vue'
|
import Button from '@/components/ui/button/Button.vue'
|
||||||
|
import Skeleton from '@/components/ui/skeleton/Skeleton.vue'
|
||||||
import { useCopyToClipboard } from '@/composables/useCopyToClipboard'
|
import { useCopyToClipboard } from '@/composables/useCopyToClipboard'
|
||||||
import { formatSize } from '@/utils/formatUtil'
|
import { formatSize } from '@/utils/formatUtil'
|
||||||
|
|
||||||
import type { ModelWithUrl } from './missingModelsUtils'
|
import type { ModelWithUrl } from './missingModelsUtils'
|
||||||
import {
|
import {
|
||||||
downloadModel,
|
downloadModel,
|
||||||
|
fetchModelMetadata,
|
||||||
getBadgeLabel,
|
getBadgeLabel,
|
||||||
hasValidDirectory,
|
hasValidDirectory,
|
||||||
isModelDownloadable
|
isModelDownloadable
|
||||||
@@ -142,6 +154,7 @@ const hasCustomModels = computed(() =>
|
|||||||
processedModels.value.some((m) => !m.isDownloadable)
|
processedModels.value.some((m) => !m.isDownloadable)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const loading = ref(true)
|
||||||
const fileSizes = reactive(new Map<string, number>())
|
const fileSizes = reactive(new Map<string, number>())
|
||||||
|
|
||||||
const totalDownloadSize = computed(() =>
|
const totalDownloadSize = computed(() =>
|
||||||
@@ -150,6 +163,17 @@ const totalDownloadSize = computed(() =>
|
|||||||
.reduce((total, model) => total + (fileSizes.get(model.url) ?? 0), 0)
|
.reduce((total, model) => total + (fileSizes.get(model.url) ?? 0), 0)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const gatedModelUrls = reactive(new Map<string, string>())
|
||||||
|
|
||||||
|
function showSkeleton(model: ProcessedModel): boolean {
|
||||||
|
return (
|
||||||
|
model.isDownloadable &&
|
||||||
|
loading.value &&
|
||||||
|
!fileSizes.has(model.url) &&
|
||||||
|
!gatedModelUrls.has(model.url)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
const downloadableUrls = processedModels.value
|
const downloadableUrls = processedModels.value
|
||||||
.filter((m) => m.isDownloadable)
|
.filter((m) => m.isDownloadable)
|
||||||
@@ -157,16 +181,12 @@ onMounted(async () => {
|
|||||||
|
|
||||||
await Promise.allSettled(
|
await Promise.allSettled(
|
||||||
downloadableUrls.map(async (url) => {
|
downloadableUrls.map(async (url) => {
|
||||||
try {
|
const metadata = await fetchModelMetadata(url)
|
||||||
const response = await fetch(url, { method: 'HEAD' })
|
if (metadata.fileSize !== null) fileSizes.set(url, metadata.fileSize)
|
||||||
if (!response.ok) return
|
if (metadata.gatedRepoUrl) gatedModelUrls.set(url, metadata.gatedRepoUrl)
|
||||||
const size = response.headers.get('content-length')
|
|
||||||
if (size) fileSizes.set(url, parseInt(size, 10))
|
|
||||||
} catch {
|
|
||||||
// Silently skip size fetch failures
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
|
loading.value = false
|
||||||
})
|
})
|
||||||
|
|
||||||
const { copyToClipboard } = useCopyToClipboard()
|
const { copyToClipboard } = useCopyToClipboard()
|
||||||
|
|||||||
142
src/components/dialog/content/missingModelsUtils.test.ts
Normal file
142
src/components/dialog/content/missingModelsUtils.test.ts
Normal file
@@ -0,0 +1,142 @@
|
|||||||
|
import { describe, expect, it, vi, beforeEach } from 'vitest'
|
||||||
|
|
||||||
|
import { fetchModelMetadata } from './missingModelsUtils'
|
||||||
|
|
||||||
|
const fetchMock = vi.fn()
|
||||||
|
vi.stubGlobal('fetch', fetchMock)
|
||||||
|
|
||||||
|
vi.mock('@/platform/distribution/types', () => ({ isDesktop: false }))
|
||||||
|
vi.mock('@/stores/electronDownloadStore', () => ({}))
|
||||||
|
|
||||||
|
let testId = 0
|
||||||
|
|
||||||
|
describe('fetchModelMetadata', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
fetchMock.mockReset()
|
||||||
|
testId++
|
||||||
|
})
|
||||||
|
|
||||||
|
it('fetches file size via HEAD for non-Civitai URLs', async () => {
|
||||||
|
fetchMock.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
headers: new Headers({ 'content-length': '1048576' })
|
||||||
|
})
|
||||||
|
|
||||||
|
const url = `https://huggingface.co/org/model/resolve/main/head-${testId}.safetensors`
|
||||||
|
const metadata = await fetchModelMetadata(url)
|
||||||
|
expect(metadata.fileSize).toBe(1048576)
|
||||||
|
expect(metadata.gatedRepoUrl).toBeNull()
|
||||||
|
expect(fetchMock).toHaveBeenCalledWith(url, { method: 'HEAD' })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('uses Civitai API for Civitai model URLs', async () => {
|
||||||
|
const url = `https://civitai.com/api/download/models/${testId}`
|
||||||
|
fetchMock.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({
|
||||||
|
files: [{ sizeKB: 1024, downloadUrl: url }]
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
const metadata = await fetchModelMetadata(url)
|
||||||
|
expect(metadata.fileSize).toBe(1024 * 1024)
|
||||||
|
expect(metadata.gatedRepoUrl).toBeNull()
|
||||||
|
expect(fetchMock).toHaveBeenCalledWith(
|
||||||
|
`https://civitai.com/api/v1/model-versions/${testId}`
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('returns null fileSize when Civitai API fails', async () => {
|
||||||
|
fetchMock.mockResolvedValueOnce({ ok: false })
|
||||||
|
|
||||||
|
const metadata = await fetchModelMetadata(
|
||||||
|
`https://civitai.com/api/download/models/${testId}`
|
||||||
|
)
|
||||||
|
expect(metadata.fileSize).toBeNull()
|
||||||
|
expect(metadata.gatedRepoUrl).toBeNull()
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('returns gatedRepoUrl for gated HuggingFace HEAD requests (403)', async () => {
|
||||||
|
fetchMock.mockResolvedValueOnce({ ok: false, status: 403 })
|
||||||
|
|
||||||
|
const metadata = await fetchModelMetadata(
|
||||||
|
`https://huggingface.co/bfl/FLUX.1/resolve/main/gated-${testId}.safetensors`
|
||||||
|
)
|
||||||
|
expect(metadata.gatedRepoUrl).toBe('https://huggingface.co/bfl/FLUX.1')
|
||||||
|
expect(metadata.fileSize).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('does not treat HuggingFace 404/500 as gated', async () => {
|
||||||
|
fetchMock.mockResolvedValueOnce({ ok: false, status: 404 })
|
||||||
|
|
||||||
|
const metadata = await fetchModelMetadata(
|
||||||
|
`https://huggingface.co/org/model/resolve/main/notfound-${testId}.safetensors`
|
||||||
|
)
|
||||||
|
expect(metadata.gatedRepoUrl).toBeNull()
|
||||||
|
expect(metadata.fileSize).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('returns null for unrecognized Civitai URL patterns', async () => {
|
||||||
|
const url = `https://civitai.com/api/v1/models/${testId}`
|
||||||
|
const metadata = await fetchModelMetadata(url)
|
||||||
|
expect(metadata.fileSize).toBeNull()
|
||||||
|
expect(metadata.gatedRepoUrl).toBeNull()
|
||||||
|
expect(fetchMock).not.toHaveBeenCalled()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('returns cached metadata on second call', async () => {
|
||||||
|
const url = `https://huggingface.co/org/model/resolve/main/cached-${testId}.safetensors`
|
||||||
|
|
||||||
|
fetchMock.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
headers: new Headers({ 'content-length': '500' })
|
||||||
|
})
|
||||||
|
|
||||||
|
const first = await fetchModelMetadata(url)
|
||||||
|
const second = await fetchModelMetadata(url)
|
||||||
|
|
||||||
|
expect(first.fileSize).toBe(500)
|
||||||
|
expect(second.fileSize).toBe(500)
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('does not cache incomplete results so retries are possible', async () => {
|
||||||
|
const url = `https://example.com/retry-${testId}.safetensors`
|
||||||
|
|
||||||
|
fetchMock
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
headers: new Headers({})
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
headers: new Headers({ 'content-length': '1024' })
|
||||||
|
})
|
||||||
|
|
||||||
|
const first = await fetchModelMetadata(url)
|
||||||
|
const second = await fetchModelMetadata(url)
|
||||||
|
|
||||||
|
expect(first.fileSize).toBeNull()
|
||||||
|
expect(second.fileSize).toBe(1024)
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(2)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('deduplicates concurrent requests for the same URL', async () => {
|
||||||
|
const url = `https://huggingface.co/org/model/resolve/main/dedup-${testId}.safetensors`
|
||||||
|
|
||||||
|
fetchMock.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
headers: new Headers({ 'content-length': '2048' })
|
||||||
|
})
|
||||||
|
|
||||||
|
const [first, second] = await Promise.all([
|
||||||
|
fetchModelMetadata(url),
|
||||||
|
fetchModelMetadata(url)
|
||||||
|
])
|
||||||
|
|
||||||
|
expect(first.fileSize).toBe(2048)
|
||||||
|
expect(second.fileSize).toBe(2048)
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { downloadUrlToHfRepoUrl, isCivitaiModelUrl } from '@/utils/formatUtil'
|
||||||
import { isDesktop } from '@/platform/distribution/types'
|
import { isDesktop } from '@/platform/distribution/types'
|
||||||
import { useElectronDownloadStore } from '@/stores/electronDownloadStore'
|
import { useElectronDownloadStore } from '@/stores/electronDownloadStore'
|
||||||
|
|
||||||
@@ -81,3 +82,99 @@ export function downloadModel(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface ModelMetadata {
|
||||||
|
fileSize: number | null
|
||||||
|
gatedRepoUrl: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CivitaiModelFile {
|
||||||
|
sizeKB: number
|
||||||
|
downloadUrl: string
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CivitaiModelVersionResponse {
|
||||||
|
files: CivitaiModelFile[]
|
||||||
|
}
|
||||||
|
|
||||||
|
const metadataCache = new Map<string, ModelMetadata>()
|
||||||
|
const inflight = new Map<string, Promise<ModelMetadata>>()
|
||||||
|
|
||||||
|
async function fetchCivitaiMetadata(url: string): Promise<ModelMetadata> {
|
||||||
|
try {
|
||||||
|
const pathname = new URL(url).pathname
|
||||||
|
const versionIdMatch =
|
||||||
|
pathname.match(/^\/api\/download\/models\/(\d+)$/) ??
|
||||||
|
pathname.match(/^\/api\/v1\/models-versions\/(\d+)$/)
|
||||||
|
|
||||||
|
if (!versionIdMatch) return { fileSize: null, gatedRepoUrl: null }
|
||||||
|
|
||||||
|
const [, modelVersionId] = versionIdMatch
|
||||||
|
const apiUrl = `https://civitai.com/api/v1/model-versions/${modelVersionId}`
|
||||||
|
const res = await fetch(apiUrl)
|
||||||
|
if (!res.ok) return { fileSize: null, gatedRepoUrl: null }
|
||||||
|
|
||||||
|
const data: CivitaiModelVersionResponse = await res.json()
|
||||||
|
const matchingFile = data.files?.find(
|
||||||
|
(file) => file.downloadUrl && file.downloadUrl.startsWith(url)
|
||||||
|
)
|
||||||
|
const fileSize = matchingFile?.sizeKB ? matchingFile.sizeKB * 1024 : null
|
||||||
|
return { fileSize, gatedRepoUrl: null }
|
||||||
|
} catch {
|
||||||
|
return { fileSize: null, gatedRepoUrl: null }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const GATED_STATUS_CODES = new Set([401, 403, 451])
|
||||||
|
|
||||||
|
async function fetchHeadMetadata(url: string): Promise<ModelMetadata> {
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, { method: 'HEAD' })
|
||||||
|
if (!response.ok) {
|
||||||
|
if (
|
||||||
|
url.includes('huggingface.co') &&
|
||||||
|
GATED_STATUS_CODES.has(response.status)
|
||||||
|
) {
|
||||||
|
return { fileSize: null, gatedRepoUrl: downloadUrlToHfRepoUrl(url) }
|
||||||
|
}
|
||||||
|
return { fileSize: null, gatedRepoUrl: null }
|
||||||
|
}
|
||||||
|
const size = response.headers.get('content-length')
|
||||||
|
return {
|
||||||
|
fileSize: size ? parseInt(size, 10) : null,
|
||||||
|
gatedRepoUrl: null
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
return { fileSize: null, gatedRepoUrl: null }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isComplete(metadata: ModelMetadata): boolean {
|
||||||
|
return metadata.fileSize !== null || metadata.gatedRepoUrl !== null
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchModelMetadata(url: string): Promise<ModelMetadata> {
|
||||||
|
const cached = metadataCache.get(url)
|
||||||
|
if (cached !== undefined) return cached
|
||||||
|
|
||||||
|
const existing = inflight.get(url)
|
||||||
|
if (existing) return existing
|
||||||
|
|
||||||
|
const promise = (async () => {
|
||||||
|
const metadata = isCivitaiModelUrl(url)
|
||||||
|
? await fetchCivitaiMetadata(url)
|
||||||
|
: await fetchHeadMetadata(url)
|
||||||
|
|
||||||
|
if (isComplete(metadata)) {
|
||||||
|
metadataCache.set(url, metadata)
|
||||||
|
}
|
||||||
|
return metadata
|
||||||
|
})()
|
||||||
|
|
||||||
|
inflight.set(url, promise)
|
||||||
|
try {
|
||||||
|
return await promise
|
||||||
|
} finally {
|
||||||
|
inflight.delete(url)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1788,7 +1788,8 @@
|
|||||||
"gotIt": "Ok, got it",
|
"gotIt": "Ok, got it",
|
||||||
"footerDescription": "Download and place these models in the correct folder.\nNodes with missing models are highlighted red on the canvas.",
|
"footerDescription": "Download and place these models in the correct folder.\nNodes with missing models are highlighted red on the canvas.",
|
||||||
"customModelsWarning": "Some of these are custom models that we don't recognize.",
|
"customModelsWarning": "Some of these are custom models that we don't recognize.",
|
||||||
"customModelsInstruction": "You'll need to find and download them manually. Search for them online (try Civitai or HuggingFace) or contact the original workflow provider."
|
"customModelsInstruction": "You'll need to find and download them manually. Search for them online (try Civitai or HuggingFace) or contact the original workflow provider.",
|
||||||
|
"acceptTerms": "Accept terms"
|
||||||
},
|
},
|
||||||
"versionMismatchWarning": {
|
"versionMismatchWarning": {
|
||||||
"title": "Version Compatibility Warning",
|
"title": "Version Compatibility Warning",
|
||||||
|
|||||||
Reference in New Issue
Block a user