diff --git a/OAI/types/common.py b/OAI/types/common.py index a6b2381..5040c71 100644 --- a/OAI/types/common.py +++ b/OAI/types/common.py @@ -74,7 +74,7 @@ class CommonCompletionRequest(BaseModel): mirostat_eta: Optional[float] = 0.1 add_bos_token: Optional[bool] = True ban_eos_token: Optional[bool] = False - logit_bias: Optional[Dict[int, float]] = None + logit_bias: Optional[Dict[int, float]] = Field(default=None, examples=[[{"1": 10}]]) # Aliased variables penalty_range: Optional[int] = Field( diff --git a/OAI/types/model.py b/OAI/types/model.py index 0951ebd..bfd395e 100644 --- a/OAI/types/model.py +++ b/OAI/types/model.py @@ -68,9 +68,15 @@ class ModelLoadRequest(BaseModel): ) gpu_split_auto: Optional[bool] = True gpu_split: Optional[List[float]] = Field(default_factory=list) - rope_scale: Optional[float] = 1.0 + rope_scale: Optional[float] = Field( + description="Automatically pulled from the model's config if not present", + default=None, + examples=[1.0], + ) rope_alpha: Optional[float] = Field( - description="Automatically calculated if not present", default=None + description="Automatically calculated if not present", + default=None, + examples=[1.0], ) no_flash_attention: Optional[bool] = False # low_mem: Optional[bool] = False diff --git a/main.py b/main.py index 7f6822f..2f8484e 100644 --- a/main.py +++ b/main.py @@ -51,7 +51,14 @@ from logger import init_logger logger = init_logger(__name__) -app = FastAPI() +app = FastAPI( + title="TabbyAPI", + summary="An OAI compatible exllamav2 API that's both lightweight and fast", + description=( + "This docs page is not meant to send requests! Please use a service " + "like Postman or a frontend UI." + ), +) # Globally scoped variables. Undefined until initalized in main MODEL_CONTAINER: Optional[ModelContainer] = None