theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-03-15 00:07:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Model: Add exl3 and associated load functions

Browse Source 
Initial exl3 compat and loading functionality.

Signed-off-by: kingbri <8082010+kingbri1@users.noreply.github.com>
This commit is contained in:
kingbri
2025-04-28 23:54:55 -04:00
parent 7c6a053747
commit 0c1d794390
5 changed files with 357 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
common/optional_dependencies.py
View File

@@ -13,6 +13,7 @@ class DependenciesModel(BaseModel):
torch: bool
exllamav2: bool
exllamav3: bool
flash_attn: bool
infinity_emb: bool
sentence_transformers: bool
@@ -25,7 +26,7 @@ class DependenciesModel(BaseModel):
@computed_field
@property
def inference(self) -> bool:
return self.torch and self.exllamav2 and self.flash_attn
return self.torch and (self.exllamav2 or self.exllamav3) and self.flash_attn
def is_installed(package_name: str) -> bool: