This commit is contained in:
Richard Yu
2025-09-02 21:12:16 -07:00
parent d83af149b0
commit f3e5b281dd
4 changed files with 428 additions and 15 deletions

View File

@@ -92,8 +92,8 @@ function applyFilenameMappingToWidget(
const hashValue = widget.value
if (typeof hashValue !== 'string') return String(hashValue)
// Try to get human-readable name from cache
const mapping = fileNameMappingService.getCachedMapping('input')
// Try to get human-readable name from cache (deduplicated for display)
const mapping = fileNameMappingService.getCachedMapping('input', true)
const humanName = mapping[hashValue]
// Return human name for display, fallback to hash
@@ -117,8 +117,8 @@ function applyFilenameMappingToWidget(
get() {
if (!Array.isArray(rawValues)) return rawValues
// Map values to human-readable names
const mapping = fileNameMappingService.getCachedMapping('input')
// Map values to human-readable names (deduplicated for dropdown display)
const mapping = fileNameMappingService.getCachedMapping('input', true)
const mapped = rawValues.map((value: any) => {
if (typeof value === 'string') {
const humanName = mapping[value]
@@ -165,8 +165,8 @@ function applyFilenameMappingToWidget(
// Override incrementValue and decrementValue for arrow key navigation
;(widget as any).incrementValue = function (options: any) {
// Get the current human-readable value
const mapping = fileNameMappingService.getCachedMapping('input')
// Get the current human-readable value (deduplicated)
const mapping = fileNameMappingService.getCachedMapping('input', true)
const currentHumanName = mapping[widget.value] || widget.value
// Get the values array (which contains human names through our proxy)
@@ -185,8 +185,8 @@ function applyFilenameMappingToWidget(
}
}
;(widget as any).decrementValue = function (options: any) {
// Get the current human-readable value
const mapping = fileNameMappingService.getCachedMapping('input')
// Get the current human-readable value (deduplicated)
const mapping = fileNameMappingService.getCachedMapping('input', true)
const currentHumanName = mapping[widget.value] || widget.value
// Get the values array (which contains human names through our proxy)
@@ -210,8 +210,11 @@ function applyFilenameMappingToWidget(
;(widget as any).setValue = function (selectedValue: any, options?: any) {
if (typeof selectedValue === 'string') {
// Check if this is a human-readable name that needs reverse mapping
const reverseMapping =
fileNameMappingService.getCachedReverseMapping('input')
// Use deduplicated reverse mapping to handle suffixed names
const reverseMapping = fileNameMappingService.getCachedReverseMapping(
'input',
true
)
const hashValue = reverseMapping[selectedValue] || selectedValue
// Set the hash value
@@ -242,8 +245,11 @@ function applyFilenameMappingToWidget(
widget.callback = function (selectedValue: any) {
if (typeof selectedValue === 'string') {
// Check if this is a human-readable name that needs reverse mapping
const reverseMapping =
fileNameMappingService.getCachedReverseMapping('input')
// Use deduplicated reverse mapping to handle suffixed names
const reverseMapping = fileNameMappingService.getCachedReverseMapping(
'input',
true
)
const hashValue = reverseMapping[selectedValue] || selectedValue
// Set the hash value

View File

@@ -8,6 +8,7 @@ export interface FileNameMapping {
export interface CacheEntry {
data: FileNameMapping
dedupData?: FileNameMapping // Deduplicated mapping with unique display names
timestamp: number
error?: Error | null
fetchPromise?: Promise<FileNameMapping>
@@ -88,16 +89,25 @@ export class FileNameMappingService {
/**
* Get cached mapping synchronously (returns empty object if not cached).
* @param fileType - The file type to get cached mapping for
* @param deduplicated - Whether to return deduplicated names for display
* @returns The cached mapping or empty object
*/
getCachedMapping(fileType: FileType = 'input'): FileNameMapping {
getCachedMapping(
fileType: FileType = 'input',
deduplicated: boolean = false
): FileNameMapping {
const cached = this.cache.get(fileType)
if (cached && !this.isExpired(cached) && !cached.failed) {
// Return deduplicated mapping if requested and available
if (deduplicated && cached.dedupData) {
return cached.dedupData
}
const result = cached.data
console.debug(
`[FileNameMapping] getCachedMapping returning cached data:`,
{
fileType,
deduplicated,
mappingCount: Object.keys(result).length,
sampleMappings: Object.entries(result).slice(0, 3)
}
@@ -113,12 +123,14 @@ export class FileNameMappingService {
/**
* Get reverse mapping (human-readable name to hash) synchronously.
* @param fileType - The file type to get reverse mapping for
* @param deduplicated - Whether to use deduplicated names
* @returns The reverse mapping object
*/
getCachedReverseMapping(
fileType: FileType = 'input'
fileType: FileType = 'input',
deduplicated: boolean = false
): Record<string, string> {
const mapping = this.getCachedMapping(fileType)
const mapping = this.getCachedMapping(fileType, deduplicated)
const reverseMapping: Record<string, string> = {}
// Build reverse mapping: humanName -> hashName
@@ -212,6 +224,7 @@ export class FileNameMappingService {
// Update cache with successful result
entry.data = data
entry.dedupData = this.deduplicateMapping(data)
entry.timestamp = Date.now()
entry.error = null
entry.failed = false
@@ -315,6 +328,73 @@ export class FileNameMappingService {
// Allow retry after 30 seconds for failed requests
return entry.timestamp > 0 && Date.now() - entry.timestamp > 30000
}
/**
* Deduplicate human-readable names when multiple hashes map to the same name.
* Adds a suffix to duplicate names to make them unique.
* @param mapping - The original hash -> human name mapping
* @returns A new mapping with deduplicated human names
*/
private deduplicateMapping(mapping: FileNameMapping): FileNameMapping {
const dedupMapping: FileNameMapping = {}
const nameCount = new Map<string, number>()
const nameToHashes = new Map<string, string[]>()
// First pass: count occurrences of each human name
for (const [hash, humanName] of Object.entries(mapping)) {
const count = nameCount.get(humanName) || 0
nameCount.set(humanName, count + 1)
// Track which hashes map to this human name
const hashes = nameToHashes.get(humanName) || []
hashes.push(hash)
nameToHashes.set(humanName, hashes)
}
// Second pass: create deduplicated names
const nameIndex = new Map<string, number>()
for (const [hash, humanName] of Object.entries(mapping)) {
const count = nameCount.get(humanName) || 1
if (count === 1) {
// No duplicates, use original name
dedupMapping[hash] = humanName
} else {
// Has duplicates, add suffix
const currentIndex = (nameIndex.get(humanName) || 0) + 1
nameIndex.set(humanName, currentIndex)
// Extract file extension if present
const lastDotIndex = humanName.lastIndexOf('.')
let baseName = humanName
let extension = ''
if (lastDotIndex > 0 && lastDotIndex < humanName.length - 1) {
baseName = humanName.substring(0, lastDotIndex)
extension = humanName.substring(lastDotIndex)
}
// Add suffix: use first 8 chars of hash (without extension)
// Remove extension from hash if present
const hashWithoutExt = hash.includes('.')
? hash.substring(0, hash.lastIndexOf('.'))
: hash
const hashSuffix = hashWithoutExt.substring(0, 8)
dedupMapping[hash] = `${baseName}_${hashSuffix}${extension}`
}
}
console.debug('[FileNameMappingService] Deduplicated mapping:', {
original: Object.keys(mapping).length,
duplicates: Array.from(nameCount.entries()).filter(
([_, count]) => count > 1
),
sample: Object.entries(dedupMapping).slice(0, 5)
})
return dedupMapping
}
}
// Singleton instance

View File

@@ -33,6 +33,158 @@ describe('useComboWidget', () => {
vi.clearAllMocks()
})
describe('deduplication', () => {
it('should display deduplicated names in dropdown', () => {
const constructor = useComboWidget()
const mockWidget = {
name: 'image',
value: 'hash1.png',
options: {
values: ['hash1.png', 'hash2.png', 'hash3.png']
},
callback: vi.fn()
}
const mockNode = {
addWidget: vi.fn().mockReturnValue(mockWidget)
}
// Mock deduplicated mapping
vi.mocked(fileNameMappingService.getCachedMapping).mockImplementation(
(_fileType, deduplicated) => {
if (deduplicated) {
return {
'hash1.png': 'vacation_hash1.png',
'hash2.png': 'vacation_hash2.png',
'hash3.png': 'landscape.png'
}
}
return {
'hash1.png': 'vacation.png',
'hash2.png': 'vacation.png',
'hash3.png': 'landscape.png'
}
}
)
const inputSpec: InputSpec = {
type: 'COMBO',
name: 'image',
options: ['hash1.png', 'hash2.png', 'hash3.png']
}
const widget = constructor(mockNode as any, inputSpec)
// Check that dropdown values are deduplicated
const dropdownValues = widget.options.values
expect(dropdownValues).toEqual([
'vacation_hash1.png',
'vacation_hash2.png',
'landscape.png'
])
})
it('should correctly handle selection of deduplicated names', () => {
const constructor = useComboWidget()
const mockWidget = {
name: 'image',
value: 'hash1.png',
options: {
values: ['hash1.png', 'hash2.png']
},
callback: vi.fn()
}
const mockNode = {
addWidget: vi.fn().mockReturnValue(mockWidget)
}
// Mock deduplicated mappings
vi.mocked(fileNameMappingService.getCachedMapping).mockImplementation(
(_fileType, deduplicated) => {
if (deduplicated) {
return {
'hash1.png': 'image_hash1.png',
'hash2.png': 'image_hash2.png'
}
}
return {
'hash1.png': 'image.png',
'hash2.png': 'image.png'
}
}
)
vi.mocked(
fileNameMappingService.getCachedReverseMapping
).mockImplementation((_fileType, deduplicated) => {
if (deduplicated) {
return {
'image_hash1.png': 'hash1.png',
'image_hash2.png': 'hash2.png'
} as Record<string, string>
}
return {
'image.png': 'hash2.png' // Last one wins in non-dedup
} as Record<string, string>
})
const inputSpec: InputSpec = {
type: 'COMBO',
name: 'image',
options: ['hash1.png', 'hash2.png']
}
const widget = constructor(mockNode as any, inputSpec)
// Select deduplicated name
;(widget as any).setValue('image_hash1.png')
// Should set the correct hash value
expect(widget.value).toBe('hash1.png')
})
it('should display correct deduplicated name in _displayValue', () => {
const constructor = useComboWidget()
const mockWidget = {
name: 'image',
value: 'abc123.png',
options: {
values: ['abc123.png', 'def456.png']
},
callback: vi.fn()
}
const mockNode = {
addWidget: vi.fn().mockReturnValue(mockWidget)
}
// Mock deduplicated mapping
vi.mocked(fileNameMappingService.getCachedMapping).mockImplementation(
(_fileType, deduplicated) => {
if (deduplicated) {
return {
'abc123.png': 'photo_abc123.png',
'def456.png': 'photo_def456.png'
}
}
return {
'abc123.png': 'photo.png',
'def456.png': 'photo.png'
}
}
)
const inputSpec: InputSpec = {
type: 'COMBO',
name: 'image',
options: ['abc123.png', 'def456.png']
}
const widget = constructor(mockNode as any, inputSpec)
// Check display value shows deduplicated name
expect((widget as any)._displayValue).toBe('photo_abc123.png')
})
})
it('should handle undefined spec', () => {
const constructor = useComboWidget()
const mockNode = {

View File

@@ -22,6 +22,181 @@ describe('FileNameMappingService', () => {
service = new FileNameMappingService()
})
describe('deduplication', () => {
it('should not modify unique names', async () => {
const mockData: FileNameMapping = {
'abc123.png': 'vacation.png',
'def456.jpg': 'profile.jpg',
'ghi789.gif': 'animation.gif'
}
vi.mocked(api.fetchApi).mockResolvedValue({
ok: true,
status: 200,
json: async () => mockData
} as any)
await service.getMapping('input')
const dedupMapping = service.getCachedMapping('input', true)
// All unique names should remain unchanged
expect(dedupMapping['abc123.png']).toBe('vacation.png')
expect(dedupMapping['def456.jpg']).toBe('profile.jpg')
expect(dedupMapping['ghi789.gif']).toBe('animation.gif')
})
it('should add hash suffix to duplicate names', async () => {
const mockData: FileNameMapping = {
'abc123def456.png': 'vacation.png',
'xyz789uvw012.png': 'vacation.png',
'mno345pqr678.png': 'vacation.png'
}
vi.mocked(api.fetchApi).mockResolvedValue({
ok: true,
status: 200,
json: async () => mockData
} as any)
await service.getMapping('input')
const dedupMapping = service.getCachedMapping('input', true)
// Check that all values are unique
const values = Object.values(dedupMapping)
const uniqueValues = new Set(values)
expect(uniqueValues.size).toBe(values.length)
// Check that suffixes are added correctly
expect(dedupMapping['abc123def456.png']).toBe('vacation_abc123de.png')
expect(dedupMapping['xyz789uvw012.png']).toBe('vacation_xyz789uv.png')
expect(dedupMapping['mno345pqr678.png']).toBe('vacation_mno345pq.png')
})
it('should preserve file extensions when deduplicating', async () => {
const mockData: FileNameMapping = {
'hash1234.safetensors': 'model.safetensors',
'hash5678.safetensors': 'model.safetensors'
}
vi.mocked(api.fetchApi).mockResolvedValue({
ok: true,
status: 200,
json: async () => mockData
} as any)
await service.getMapping('input')
const dedupMapping = service.getCachedMapping('input', true)
// Extensions should be preserved
expect(dedupMapping['hash1234.safetensors']).toBe(
'model_hash1234.safetensors'
)
expect(dedupMapping['hash5678.safetensors']).toBe(
'model_hash5678.safetensors'
)
})
it('should handle files without extensions', async () => {
const mockData: FileNameMapping = {
abc123: 'README',
def456: 'README',
ghi789: 'LICENSE'
}
vi.mocked(api.fetchApi).mockResolvedValue({
ok: true,
status: 200,
json: async () => mockData
} as any)
await service.getMapping('input')
const dedupMapping = service.getCachedMapping('input', true)
// Files without extensions should still get deduplicated
expect(dedupMapping['abc123']).toBe('README_abc123')
expect(dedupMapping['def456']).toBe('README_def456')
expect(dedupMapping['ghi789']).toBe('LICENSE') // Unique, no suffix
})
it('should build correct reverse mapping for deduplicated names', async () => {
const mockData: FileNameMapping = {
'hash1.png': 'image.png',
'hash2.png': 'image.png',
'hash3.jpg': 'photo.jpg'
}
vi.mocked(api.fetchApi).mockResolvedValue({
ok: true,
status: 200,
json: async () => mockData
} as any)
await service.getMapping('input')
const reverseMapping = service.getCachedReverseMapping('input', true)
// Reverse mapping should map deduplicated names back to hashes
expect(reverseMapping['image_hash1.png']).toBe('hash1.png')
expect(reverseMapping['image_hash2.png']).toBe('hash2.png')
expect(reverseMapping['photo.jpg']).toBe('hash3.jpg')
// Should not have original duplicate names in reverse mapping
expect(reverseMapping['image.png']).toBeUndefined()
})
it('should handle mixed duplicate and unique names', async () => {
const mockData: FileNameMapping = {
'a1.png': 'sunset.png',
'b2.png': 'sunset.png',
'c3.jpg': 'portrait.jpg',
'd4.gif': 'animation.gif',
'e5.png': 'sunset.png'
}
vi.mocked(api.fetchApi).mockResolvedValue({
ok: true,
status: 200,
json: async () => mockData
} as any)
await service.getMapping('input')
const dedupMapping = service.getCachedMapping('input', true)
// Duplicates get suffixes
expect(dedupMapping['a1.png']).toBe('sunset_a1.png')
expect(dedupMapping['b2.png']).toBe('sunset_b2.png')
expect(dedupMapping['e5.png']).toBe('sunset_e5.png')
// Unique names remain unchanged
expect(dedupMapping['c3.jpg']).toBe('portrait.jpg')
expect(dedupMapping['d4.gif']).toBe('animation.gif')
})
it('should return non-deduplicated mapping when deduplicated=false', async () => {
const mockData: FileNameMapping = {
'hash1.png': 'image.png',
'hash2.png': 'image.png'
}
vi.mocked(api.fetchApi).mockResolvedValue({
ok: true,
status: 200,
json: async () => mockData
} as any)
await service.getMapping('input')
// Without deduplication flag
const normalMapping = service.getCachedMapping('input', false)
expect(normalMapping['hash1.png']).toBe('image.png')
expect(normalMapping['hash2.png']).toBe('image.png')
// With deduplication flag
const dedupMapping = service.getCachedMapping('input', true)
expect(dedupMapping['hash1.png']).toBe('image_hash1.png')
expect(dedupMapping['hash2.png']).toBe('image_hash2.png')
})
})
describe('getMapping', () => {
it('should fetch mappings from API', async () => {
const mockData: FileNameMapping = {