theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-20 14:28:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sampling: Update DRY

Browse Source 
Switch to new parameters and remove dry_max_ngram as that's not supposed
to be changed.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri
2024-09-07 12:39:14 -04:00
parent 05c3f1194f
commit ae37f3f332
2 changed files with 16 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
backends/exllamav2/model.py
View File

@@ -1081,14 +1081,21 @@ class ExllamaV2Container:
)
# DRY options
dry_allowed_length = unwrap(kwargs.get("dry_allowed_length"), 0)
dry_multiplier = unwrap(kwargs.get("dry_multiplier"), 0.0)
# 0 = disabled
if dry_allowed_length:
gen_settings.dry_allowed_length = dry_allowed_length
# < 0 = disabled
if dry_multiplier > 0:
gen_settings.dry_allowed_length = unwrap(
kwargs.get("dry_allowed_length"), 0
)
gen_settings.dry_base = unwrap(kwargs.get("dry_base"), 2.0)
gen_settings.dry_multiplier = unwrap(kwargs.get("dry_multiplier"), 2.0)
gen_settings.dry_max_ngram = unwrap(kwargs.get("dry_max_ngram"), 20)
# Exl2 has dry_range as 0 for unlimited unlike -1 for penalty_range
# Use max_seq_len as the fallback to stay consistent
gen_settings.dry_range = unwrap(
kwargs.get("dry_range"), self.config.max_seq_len
)
# Tokenize sequence breakers
dry_sequence_breakers_json = kwargs.get("dry_sequence_breakers")