theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-27 01:38:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tree: Format

Browse Source
This commit is contained in:
turboderp
2025-06-15 19:30:38 +02:00
parent 1c9891bf04
commit 21c5af48e1
2 changed files with 7 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backends/exllamav3/model.py
View File

@@ -435,7 +435,7 @@ class ExllamaV3Container(BaseModelContainer):
if self.use_vision: if self.use_vision:
for value in self.vision_model.load_gen( for value in self.vision_model.load_gen(
reserve_per_device=self.autosplit_reserve, reserve_per_device=self.autosplit_reserve,
callback=progress_callback callback=progress_callback,
): ):
if value: if value:
yield value yield value
@@ -559,7 +559,7 @@ class ExllamaV3Container(BaseModelContainer):
kwargs.get("add_bos_token"), self.hf_model.add_bos_token() kwargs.get("add_bos_token"), self.hf_model.add_bos_token()
), ),
encode_special_tokens=unwrap(kwargs.get("encode_special_tokens"), True), encode_special_tokens=unwrap(kwargs.get("encode_special_tokens"), True),
embeddings=mm_embeddings_content embeddings=mm_embeddings_content,
) )
.flatten() .flatten()
.tolist() .tolist()

12
common/multimodal.py
View File

@@ -22,14 +22,12 @@ class MultimodalEmbeddingWrapper(BaseModel):
async def add(self, url: str): async def add(self, url: str):
# Determine the type of vision embedding to use # Determine the type of vision embedding to use
if not self.type: if not self.type:
if ( if dependencies.exllamav2 and isinstance(
dependencies.exllamav2 and model.container.vision_model, ExLlamaV2VisionTower
isinstance(model.container.vision_model, ExLlamaV2VisionTower)
): ):
self.type = "ExLlamaV2MMEmbedding" self.type = "ExLlamaV2MMEmbedding"
elif ( elif dependencies.exllamav3 and isinstance(
dependencies.exllamav3 and model.container.vision_model, Model
isinstance(model.container.vision_model, Model)
): ):
self.type = "MMEmbedding" self.type = "MMEmbedding"
@@ -43,4 +41,4 @@ class MultimodalEmbeddingWrapper(BaseModel):
self.content.append(embedding) self.content.append(embedding)
self.text_alias.append(embedding.text_alias) self.text_alias.append(embedding.text_alias)
else: else:
logger.error("No valid vision model to create embedding") logger.error("No valid vision model to create embedding")