theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-29 02:31:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tree: Format

Browse Source
This commit is contained in:
DocShotgun
2024-05-26 15:48:18 -07:00
parent 767e6a798a
commit 7ab7ffd562
2 changed files with 2 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/args.py
View File

@@ -85,7 +85,7 @@ def add_model_args(parser: argparse.ArgumentParser):
model_group.add_argument( model_group.add_argument(
"--cache-size", "--cache-size",
type=int, type=int,
help="The size of the prompt cache (in number of tokens) to allocate" help="The size of the prompt cache (in number of tokens) to allocate",
) )
model_group.add_argument( model_group.add_argument(
"--rope-scale", type=float, help="Sets rope_scale or compress_pos_emb" "--rope-scale", type=float, help="Sets rope_scale or compress_pos_emb"

4
endpoints/OAI/types/model.py
View File

@@ -74,9 +74,7 @@ class ModelLoadRequest(BaseModel):
examples=[4096], examples=[4096],
) )
cache_size: Optional[int] = Field( cache_size: Optional[int] = Field(
description=( description=("Number in tokens, must be greater than or equal to max_seq_len"),
"Number in tokens, must be greater than or equal to max_seq_len"
),
default=None, default=None,
examples=[4096], examples=[4096],
) )