theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-19 22:08:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sampling: Add top-a support

Browse Source 
Currently in exllamav2 dev, but will be in the next release.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri
2023-12-22 23:50:24 -05:00
parent 6a5bbd217c
commit 80ef379721
2 changed files with 11 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
OAI/types/common.py
View File

@@ -69,6 +69,7 @@ class CommonCompletionRequest(BaseModel):
temperature_last: Optional[bool] = False
top_k: Optional[int] = 0
top_p: Optional[float] = 1.0
top_a: Optional[float] = 0.0
typical: Optional[float] = 1.0
min_p: Optional[float] = 0.0
tfs: Optional[float] = 1.0
@@ -111,6 +112,7 @@ class CommonCompletionRequest(BaseModel):
"temperature_last": self.temperature_last,
"top_k": self.top_k,
"top_p": self.top_p,
"top_a": self.top_a,
"typical": self.typical,
"min_p": self.min_p,
"tfs": self.tfs,

9
model.py
View File

@@ -523,11 +523,20 @@ class ModelContainer:
"installed ExLlamaV2 version."
)
if (unwrap(kwargs.get("top_a"), False)) and not hasattr (
gen_settings, "top_a"
):
logger.warning(
"Top-A is not supported by the currently "
"installed ExLlamaV2 version."
)
# Apply settings
gen_settings.temperature = unwrap(kwargs.get("temperature"), 1.0)
gen_settings.temperature_last = unwrap(kwargs.get("temperature_last"), False)
gen_settings.top_k = unwrap(kwargs.get("top_k"), 0)
gen_settings.top_p = unwrap(kwargs.get("top_p"), 1.0)
gen_settings.top_a = unwrap(kwargs.get("top_a"), 0.0)
gen_settings.min_p = unwrap(kwargs.get("min_p"), 0.0)
gen_settings.tfs = unwrap(kwargs.get("tfs"), 1.0)
gen_settings.typical = unwrap(kwargs.get("typical"), 1.0)