theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-20 14:28:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

added quadratic sampling (#56)

Browse Source 
* added quadratic sampling

* Update sample_preset.yml

* oops missed a spot

* Sampling: Fix smoothing factor semantics
This commit is contained in:
Alexander Abushady
2024-02-02 22:12:59 -05:00
committed by GitHub
parent 4a7b8b1b7a
commit d7c18855e7
3 changed files with 17 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
backends/exllamav2/model.py
View File

@@ -523,6 +523,14 @@ class ExllamaV2Container:
"installed ExLlamaV2 version."
)
if (unwrap(kwargs.get("smoothing_factor"), 0.0)) > 0.0 and not hasattr(
ExLlamaV2Sampler.Settings, "smoothing_factor"
):
logger.warning(
"Smoothing factor is not supported by the currently "
"installed ExLlamaV2 version."
)
def generate(self, prompt: str, **kwargs):
"""Generate a response to a prompt"""
generation = list(self.generate_gen(prompt, **kwargs))
@@ -593,6 +601,7 @@ class ExllamaV2Container:
# Apply settings
gen_settings.temperature = unwrap(kwargs.get("temperature"), 1.0)
gen_settings.temperature_last = unwrap(kwargs.get("temperature_last"), False)
gen_settings.smoothing_factor = unwrap(kwargs.get("smoothing_factor"), 0.0)
gen_settings.top_k = unwrap(kwargs.get("top_k"), 0)
gen_settings.top_p = unwrap(kwargs.get("top_p"), 1.0)
gen_settings.top_a = unwrap(kwargs.get("top_a"), 0.0)