mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-03-07 22:20:03 +00:00
[feat] Add model metadata fetching with loading skeleton and gated repo support (#9415)
## Summary - Fetch file sizes via HEAD requests (HuggingFace) and Civitai API with caching and request deduplication - Show skeleton loader while metadata is loading - Display "Accept terms" link for gated HuggingFace models instead of download button ## Changes - **`missingModelsUtils.ts`**: Add `fetchModelMetadata` with Civitai API support, HuggingFace gated repo detection, in-memory cache, and inflight request deduplication - **`MissingModelsContent.vue`**: Add Skeleton loading state, gated model "Accept terms" link, extract `showSkeleton` helper - **`missingModelsUtils.test.ts`**: Tests for HEAD/Civitai fetching, gated repo detection, caching, and deduplication - **`main.json`**: Add `acceptTerms` i18n key ## Related Issues https://github.com/Comfy-Org/ComfyUI_frontend/issues/9410 https://github.com/Comfy-Org/ComfyUI_frontend/issues/9412 ┆Issue is synchronized with this [Notion page](https://www.notion.so/PR-9415-feat-Add-model-metadata-fetching-with-loading-skeleton-and-gated-repo-support-31a6d73d36508127859efa0b3847505e) by [Unito](https://www.unito.io)
This commit is contained in:
@@ -29,14 +29,24 @@
|
||||
</span>
|
||||
</div>
|
||||
<div class="flex shrink-0 items-center gap-2">
|
||||
<Skeleton v-if="showSkeleton(model)" class="ml-1.5 h-4 w-12" />
|
||||
<span
|
||||
v-if="model.isDownloadable && fileSizes.get(model.url)"
|
||||
class="text-xs text-muted-foreground"
|
||||
v-else-if="model.isDownloadable && fileSizes.get(model.url)"
|
||||
class="pl-1.5 text-xs text-muted-foreground"
|
||||
>
|
||||
{{ formatSize(fileSizes.get(model.url)) }}
|
||||
</span>
|
||||
<a
|
||||
v-else-if="gatedModelUrls.has(model.url)"
|
||||
:href="gatedModelUrls.get(model.url)"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="text-xs text-primary hover:underline"
|
||||
>
|
||||
{{ $t('missingModelsDialog.acceptTerms') }}
|
||||
</a>
|
||||
<Button
|
||||
v-if="model.isDownloadable"
|
||||
v-else-if="model.isDownloadable"
|
||||
variant="textonly"
|
||||
size="icon"
|
||||
:title="model.url"
|
||||
@@ -100,15 +110,17 @@
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { computed, onMounted, reactive } from 'vue'
|
||||
import { computed, onMounted, reactive, ref } from 'vue'
|
||||
|
||||
import Button from '@/components/ui/button/Button.vue'
|
||||
import Skeleton from '@/components/ui/skeleton/Skeleton.vue'
|
||||
import { useCopyToClipboard } from '@/composables/useCopyToClipboard'
|
||||
import { formatSize } from '@/utils/formatUtil'
|
||||
|
||||
import type { ModelWithUrl } from './missingModelsUtils'
|
||||
import {
|
||||
downloadModel,
|
||||
fetchModelMetadata,
|
||||
getBadgeLabel,
|
||||
hasValidDirectory,
|
||||
isModelDownloadable
|
||||
@@ -142,6 +154,7 @@ const hasCustomModels = computed(() =>
|
||||
processedModels.value.some((m) => !m.isDownloadable)
|
||||
)
|
||||
|
||||
const loading = ref(true)
|
||||
const fileSizes = reactive(new Map<string, number>())
|
||||
|
||||
const totalDownloadSize = computed(() =>
|
||||
@@ -150,6 +163,17 @@ const totalDownloadSize = computed(() =>
|
||||
.reduce((total, model) => total + (fileSizes.get(model.url) ?? 0), 0)
|
||||
)
|
||||
|
||||
const gatedModelUrls = reactive(new Map<string, string>())
|
||||
|
||||
function showSkeleton(model: ProcessedModel): boolean {
|
||||
return (
|
||||
model.isDownloadable &&
|
||||
loading.value &&
|
||||
!fileSizes.has(model.url) &&
|
||||
!gatedModelUrls.has(model.url)
|
||||
)
|
||||
}
|
||||
|
||||
onMounted(async () => {
|
||||
const downloadableUrls = processedModels.value
|
||||
.filter((m) => m.isDownloadable)
|
||||
@@ -157,16 +181,12 @@ onMounted(async () => {
|
||||
|
||||
await Promise.allSettled(
|
||||
downloadableUrls.map(async (url) => {
|
||||
try {
|
||||
const response = await fetch(url, { method: 'HEAD' })
|
||||
if (!response.ok) return
|
||||
const size = response.headers.get('content-length')
|
||||
if (size) fileSizes.set(url, parseInt(size, 10))
|
||||
} catch {
|
||||
// Silently skip size fetch failures
|
||||
}
|
||||
const metadata = await fetchModelMetadata(url)
|
||||
if (metadata.fileSize !== null) fileSizes.set(url, metadata.fileSize)
|
||||
if (metadata.gatedRepoUrl) gatedModelUrls.set(url, metadata.gatedRepoUrl)
|
||||
})
|
||||
)
|
||||
loading.value = false
|
||||
})
|
||||
|
||||
const { copyToClipboard } = useCopyToClipboard()
|
||||
|
||||
142
src/components/dialog/content/missingModelsUtils.test.ts
Normal file
142
src/components/dialog/content/missingModelsUtils.test.ts
Normal file
@@ -0,0 +1,142 @@
|
||||
import { describe, expect, it, vi, beforeEach } from 'vitest'
|
||||
|
||||
import { fetchModelMetadata } from './missingModelsUtils'
|
||||
|
||||
const fetchMock = vi.fn()
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
|
||||
vi.mock('@/platform/distribution/types', () => ({ isDesktop: false }))
|
||||
vi.mock('@/stores/electronDownloadStore', () => ({}))
|
||||
|
||||
let testId = 0
|
||||
|
||||
describe('fetchModelMetadata', () => {
|
||||
beforeEach(() => {
|
||||
fetchMock.mockReset()
|
||||
testId++
|
||||
})
|
||||
|
||||
it('fetches file size via HEAD for non-Civitai URLs', async () => {
|
||||
fetchMock.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
headers: new Headers({ 'content-length': '1048576' })
|
||||
})
|
||||
|
||||
const url = `https://huggingface.co/org/model/resolve/main/head-${testId}.safetensors`
|
||||
const metadata = await fetchModelMetadata(url)
|
||||
expect(metadata.fileSize).toBe(1048576)
|
||||
expect(metadata.gatedRepoUrl).toBeNull()
|
||||
expect(fetchMock).toHaveBeenCalledWith(url, { method: 'HEAD' })
|
||||
})
|
||||
|
||||
it('uses Civitai API for Civitai model URLs', async () => {
|
||||
const url = `https://civitai.com/api/download/models/${testId}`
|
||||
fetchMock.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
files: [{ sizeKB: 1024, downloadUrl: url }]
|
||||
})
|
||||
})
|
||||
|
||||
const metadata = await fetchModelMetadata(url)
|
||||
expect(metadata.fileSize).toBe(1024 * 1024)
|
||||
expect(metadata.gatedRepoUrl).toBeNull()
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
`https://civitai.com/api/v1/model-versions/${testId}`
|
||||
)
|
||||
})
|
||||
|
||||
it('returns null fileSize when Civitai API fails', async () => {
|
||||
fetchMock.mockResolvedValueOnce({ ok: false })
|
||||
|
||||
const metadata = await fetchModelMetadata(
|
||||
`https://civitai.com/api/download/models/${testId}`
|
||||
)
|
||||
expect(metadata.fileSize).toBeNull()
|
||||
expect(metadata.gatedRepoUrl).toBeNull()
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('returns gatedRepoUrl for gated HuggingFace HEAD requests (403)', async () => {
|
||||
fetchMock.mockResolvedValueOnce({ ok: false, status: 403 })
|
||||
|
||||
const metadata = await fetchModelMetadata(
|
||||
`https://huggingface.co/bfl/FLUX.1/resolve/main/gated-${testId}.safetensors`
|
||||
)
|
||||
expect(metadata.gatedRepoUrl).toBe('https://huggingface.co/bfl/FLUX.1')
|
||||
expect(metadata.fileSize).toBeNull()
|
||||
})
|
||||
|
||||
it('does not treat HuggingFace 404/500 as gated', async () => {
|
||||
fetchMock.mockResolvedValueOnce({ ok: false, status: 404 })
|
||||
|
||||
const metadata = await fetchModelMetadata(
|
||||
`https://huggingface.co/org/model/resolve/main/notfound-${testId}.safetensors`
|
||||
)
|
||||
expect(metadata.gatedRepoUrl).toBeNull()
|
||||
expect(metadata.fileSize).toBeNull()
|
||||
})
|
||||
|
||||
it('returns null for unrecognized Civitai URL patterns', async () => {
|
||||
const url = `https://civitai.com/api/v1/models/${testId}`
|
||||
const metadata = await fetchModelMetadata(url)
|
||||
expect(metadata.fileSize).toBeNull()
|
||||
expect(metadata.gatedRepoUrl).toBeNull()
|
||||
expect(fetchMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns cached metadata on second call', async () => {
|
||||
const url = `https://huggingface.co/org/model/resolve/main/cached-${testId}.safetensors`
|
||||
|
||||
fetchMock.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
headers: new Headers({ 'content-length': '500' })
|
||||
})
|
||||
|
||||
const first = await fetchModelMetadata(url)
|
||||
const second = await fetchModelMetadata(url)
|
||||
|
||||
expect(first.fileSize).toBe(500)
|
||||
expect(second.fileSize).toBe(500)
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('does not cache incomplete results so retries are possible', async () => {
|
||||
const url = `https://example.com/retry-${testId}.safetensors`
|
||||
|
||||
fetchMock
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
headers: new Headers({})
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
headers: new Headers({ 'content-length': '1024' })
|
||||
})
|
||||
|
||||
const first = await fetchModelMetadata(url)
|
||||
const second = await fetchModelMetadata(url)
|
||||
|
||||
expect(first.fileSize).toBeNull()
|
||||
expect(second.fileSize).toBe(1024)
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('deduplicates concurrent requests for the same URL', async () => {
|
||||
const url = `https://huggingface.co/org/model/resolve/main/dedup-${testId}.safetensors`
|
||||
|
||||
fetchMock.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
headers: new Headers({ 'content-length': '2048' })
|
||||
})
|
||||
|
||||
const [first, second] = await Promise.all([
|
||||
fetchModelMetadata(url),
|
||||
fetchModelMetadata(url)
|
||||
])
|
||||
|
||||
expect(first.fileSize).toBe(2048)
|
||||
expect(second.fileSize).toBe(2048)
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
@@ -1,3 +1,4 @@
|
||||
import { downloadUrlToHfRepoUrl, isCivitaiModelUrl } from '@/utils/formatUtil'
|
||||
import { isDesktop } from '@/platform/distribution/types'
|
||||
import { useElectronDownloadStore } from '@/stores/electronDownloadStore'
|
||||
|
||||
@@ -81,3 +82,99 @@ export function downloadModel(
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
interface ModelMetadata {
|
||||
fileSize: number | null
|
||||
gatedRepoUrl: string | null
|
||||
}
|
||||
|
||||
interface CivitaiModelFile {
|
||||
sizeKB: number
|
||||
downloadUrl: string
|
||||
}
|
||||
|
||||
interface CivitaiModelVersionResponse {
|
||||
files: CivitaiModelFile[]
|
||||
}
|
||||
|
||||
const metadataCache = new Map<string, ModelMetadata>()
|
||||
const inflight = new Map<string, Promise<ModelMetadata>>()
|
||||
|
||||
async function fetchCivitaiMetadata(url: string): Promise<ModelMetadata> {
|
||||
try {
|
||||
const pathname = new URL(url).pathname
|
||||
const versionIdMatch =
|
||||
pathname.match(/^\/api\/download\/models\/(\d+)$/) ??
|
||||
pathname.match(/^\/api\/v1\/models-versions\/(\d+)$/)
|
||||
|
||||
if (!versionIdMatch) return { fileSize: null, gatedRepoUrl: null }
|
||||
|
||||
const [, modelVersionId] = versionIdMatch
|
||||
const apiUrl = `https://civitai.com/api/v1/model-versions/${modelVersionId}`
|
||||
const res = await fetch(apiUrl)
|
||||
if (!res.ok) return { fileSize: null, gatedRepoUrl: null }
|
||||
|
||||
const data: CivitaiModelVersionResponse = await res.json()
|
||||
const matchingFile = data.files?.find(
|
||||
(file) => file.downloadUrl && file.downloadUrl.startsWith(url)
|
||||
)
|
||||
const fileSize = matchingFile?.sizeKB ? matchingFile.sizeKB * 1024 : null
|
||||
return { fileSize, gatedRepoUrl: null }
|
||||
} catch {
|
||||
return { fileSize: null, gatedRepoUrl: null }
|
||||
}
|
||||
}
|
||||
|
||||
const GATED_STATUS_CODES = new Set([401, 403, 451])
|
||||
|
||||
async function fetchHeadMetadata(url: string): Promise<ModelMetadata> {
|
||||
try {
|
||||
const response = await fetch(url, { method: 'HEAD' })
|
||||
if (!response.ok) {
|
||||
if (
|
||||
url.includes('huggingface.co') &&
|
||||
GATED_STATUS_CODES.has(response.status)
|
||||
) {
|
||||
return { fileSize: null, gatedRepoUrl: downloadUrlToHfRepoUrl(url) }
|
||||
}
|
||||
return { fileSize: null, gatedRepoUrl: null }
|
||||
}
|
||||
const size = response.headers.get('content-length')
|
||||
return {
|
||||
fileSize: size ? parseInt(size, 10) : null,
|
||||
gatedRepoUrl: null
|
||||
}
|
||||
} catch {
|
||||
return { fileSize: null, gatedRepoUrl: null }
|
||||
}
|
||||
}
|
||||
|
||||
function isComplete(metadata: ModelMetadata): boolean {
|
||||
return metadata.fileSize !== null || metadata.gatedRepoUrl !== null
|
||||
}
|
||||
|
||||
export async function fetchModelMetadata(url: string): Promise<ModelMetadata> {
|
||||
const cached = metadataCache.get(url)
|
||||
if (cached !== undefined) return cached
|
||||
|
||||
const existing = inflight.get(url)
|
||||
if (existing) return existing
|
||||
|
||||
const promise = (async () => {
|
||||
const metadata = isCivitaiModelUrl(url)
|
||||
? await fetchCivitaiMetadata(url)
|
||||
: await fetchHeadMetadata(url)
|
||||
|
||||
if (isComplete(metadata)) {
|
||||
metadataCache.set(url, metadata)
|
||||
}
|
||||
return metadata
|
||||
})()
|
||||
|
||||
inflight.set(url, promise)
|
||||
try {
|
||||
return await promise
|
||||
} finally {
|
||||
inflight.delete(url)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1788,7 +1788,8 @@
|
||||
"gotIt": "Ok, got it",
|
||||
"footerDescription": "Download and place these models in the correct folder.\nNodes with missing models are highlighted red on the canvas.",
|
||||
"customModelsWarning": "Some of these are custom models that we don't recognize.",
|
||||
"customModelsInstruction": "You'll need to find and download them manually. Search for them online (try Civitai or HuggingFace) or contact the original workflow provider."
|
||||
"customModelsInstruction": "You'll need to find and download them manually. Search for them online (try Civitai or HuggingFace) or contact the original workflow provider.",
|
||||
"acceptTerms": "Accept terms"
|
||||
},
|
||||
"versionMismatchWarning": {
|
||||
"title": "Version Compatibility Warning",
|
||||
|
||||
Reference in New Issue
Block a user