theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-26 17:28:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sampling: Add XTC support

Browse Source 
Matches with upstream.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri
2024-09-24 18:10:52 -04:00
parent f4791e7ed9
commit 56ce82ef77
3 changed files with 31 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
backends/exllamav2/model.py
View File

@@ -958,6 +958,13 @@ class ExllamaV2Container:
Meant for dev wheels!
"""
if unwrap(kwargs.get("xtc_probability"), 0.0) > 0.0 and not hasattr(
ExLlamaV2Sampler.Settings, "xtc_probability"
):
logger.warning(
"XTC is not supported by the currently " "installed ExLlamaV2 version."
)
return kwargs
async def generate_gen(
@@ -1003,6 +1010,14 @@ class ExllamaV2Container:
gen_settings.mirostat = unwrap(kwargs.get("mirostat"), False)
gen_settings.skew = unwrap(kwargs.get("skew"), 0)
# XTC
xtc_probability = unwrap(kwargs.get("xtc_probability"), 0.0)
if xtc_probability > 0.0:
gen_settings.xtc_probability = xtc_probability
# 0.1 is the default for this value
gen_settings.xtc_threshold = unwrap(kwargs.get("xtc_threshold", 0.1))
# DynaTemp settings
max_temp = unwrap(kwargs.get("max_temp"), 1.0)
min_temp = unwrap(kwargs.get("min_temp"), 1.0)