theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-28 02:01:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sampling: Update overrides and params

Browse Source 
Re-order to make more sense.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri
2024-09-07 12:48:59 -04:00
parent ae37f3f332
commit 4f5ca7a4c7
3 changed files with 25 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backends/exllamav2/model.py
View File

@@ -1085,11 +1085,13 @@ class ExllamaV2Container:
# < 0 = disabled
if dry_multiplier > 0:
gen_settings.dry_multiplier = dry_multiplier
# TODO: Maybe set the "sane" defaults instead?
gen_settings.dry_allowed_length = unwrap(
kwargs.get("dry_allowed_length"), 0
)
gen_settings.dry_base = unwrap(kwargs.get("dry_base"), 2.0)
gen_settings.dry_multiplier = unwrap(kwargs.get("dry_multiplier"), 2.0)
gen_settings.dry_base = unwrap(kwargs.get("dry_base"), 0.0)
# Exl2 has dry_range as 0 for unlimited unlike -1 for penalty_range
# Use max_seq_len as the fallback to stay consistent