From 70fbee3eddedbc205b181ef934a3688936237602 Mon Sep 17 00:00:00 2001 From: kingbri Date: Sun, 17 Dec 2023 14:36:28 -0500 Subject: [PATCH] OAI: Fix model parameter placement Accidentally edited the Model Card parameters vs the model load request ones. Signed-off-by: kingbri --- OAI/types/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OAI/types/model.py b/OAI/types/model.py index dde43fc..839d554 100644 --- a/OAI/types/model.py +++ b/OAI/types/model.py @@ -6,7 +6,7 @@ from gen_logging import LogConfig class ModelCardParameters(BaseModel): max_seq_len: Optional[int] = 4096 rope_scale: Optional[float] = 1.0 - rope_alpha: Optional[float] = None + rope_alpha: Optional[float] = 1.0 prompt_template: Optional[str] = None cache_mode: Optional[str] = "FP16" draft: Optional['ModelCard'] = None @@ -35,7 +35,7 @@ class ModelLoadRequest(BaseModel): gpu_split_auto: Optional[bool] = True gpu_split: Optional[List[float]] = Field(default_factory=list) rope_scale: Optional[float] = 1.0 - rope_alpha: Optional[float] = 1.0 + rope_alpha: Optional[float] = None no_flash_attention: Optional[bool] = False # low_mem: Optional[bool] = False cache_mode: Optional[str] = "FP16"