theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-03-14 15:57:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Config + Endpoints: Make cache_size more prominent

Browse Source 
Since cache_size is a more important parameter now for multi-user
setups, mark it as such by placing it below max_seq_len.

Signed-off-by: kingbri <8082010+kingbri1@users.noreply.github.com>
This commit is contained in:
kingbri
2025-10-14 21:53:33 -04:00
parent 62e9fa217a
commit 69a25d7fa6
3 changed files with 31 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
endpoints/core/types/model.py
View File

@@ -14,11 +14,11 @@ class ModelCardParameters(BaseModel):
# Safe to do this since it's guaranteed to fetch a max seq len
# from model_container
max_seq_len: Optional[int] = None
cache_size: Optional[int] = None
cache_mode: Optional[str] = "FP16"
rope_scale: Optional[float] = 1.0
rope_alpha: Optional[float] = 1.0
max_batch_size: Optional[int] = 1
cache_size: Optional[int] = None
cache_mode: Optional[str] = "FP16"
chunk_size: Optional[int] = 2048
prompt_template: Optional[str] = None
prompt_template_content: Optional[str] = None
@@ -89,6 +89,7 @@ class ModelLoadRequest(BaseModel):
default=None,
examples=[4096],
)
cache_mode: Optional[str] = None
tensor_parallel: Optional[bool] = None
tensor_parallel_backend: Optional[str] = "native"
gpu_split_auto: Optional[bool] = None
@@ -107,7 +108,6 @@ class ModelLoadRequest(BaseModel):
default=None,
examples=[1.0],
)
cache_mode: Optional[str] = None
chunk_size: Optional[int] = None
output_chunking: Optional[bool] = True
prompt_template: Optional[str] = None