theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-03-15 00:07:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Config: Embeddings: Make embeddings_device a default when API loading

Browse Source 
When loading from the API, the fallback for embeddings_device will be
the same as the config.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri
2024-08-01 13:59:49 -04:00
parent 54aeebaec1
commit 3e42211c3e
3 changed files with 30 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
endpoints/core/types/model.py
View File

@@ -53,19 +53,19 @@ class DraftModelLoadRequest(BaseModel):
# Config arguments
draft_rope_scale: Optional[float] = Field(
default_factory=lambda: get_config_default(
"draft_rope_scale", 1.0, is_draft=True
"draft_rope_scale", 1.0, model_type="draft"
)
)
draft_rope_alpha: Optional[float] = Field(
description="Automatically calculated if not present",
default_factory=lambda: get_config_default(
"draft_rope_alpha", None, is_draft=True
"draft_rope_alpha", None, model_type="draft"
),
examples=[1.0],
)
draft_cache_mode: Optional[str] = Field(
default_factory=lambda: get_config_default(
"draft_cache_mode", "FP16", is_draft=True
"draft_cache_mode", "FP16", model_type="draft"
)
)
@@ -139,7 +139,11 @@ class ModelLoadRequest(BaseModel):
class EmbeddingModelLoadRequest(BaseModel):
name: str
embeddings_device: Optional[str] = None
embeddings_device: Optional[str] = Field(
default_factory=lambda: get_config_default(
"embeddings_device", model_type="embedding"
)
)
class ModelLoadResponse(BaseModel):