turboderp-org/exllamav2
1
0
Fork 0
mirror of https://github.com/turboderp-org/exllamav2.git synced 2026-03-15 00:07:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Generator: Make sampler settings optional instead of default arg

Browse Source
This commit is contained in:
turboderp
2024-11-12 03:41:59 +01:00
parent 90895967b1
commit 16cd5ef384
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
exllamav2/generator/dynamic.py
View File

@@ -655,7 +655,7 @@ class ExLlamaV2DynamicGenerator:
assert False, "Unexpected type in prompt"
if gen_settings is None:
p_settings = ExLlamaV2Sampler.Settings()
p_settings = None
elif isinstance(gen_settings, ExLlamaV2Sampler.Settings):
p_settings = gen_settings
elif isinstance(gen_settings, list):
@@ -1536,7 +1536,7 @@ class ExLlamaV2DynamicJob:
max_new_tokens: int,
min_new_tokens: int = 0,
max_skips: int | None = 4,
gen_settings: ExLlamaV2Sampler.Settings = ExLlamaV2Sampler.Settings(),
gen_settings: ExLlamaV2Sampler.Settings | None = None,
seed: int = None,
stop_conditions: list | tuple | set = None,
decode_special_tokens: bool = False,
@@ -1629,6 +1629,9 @@ class ExLlamaV2DynamicJob:
self.max_skips = max_skips
self.allocated_pages = None
if gen_settings is None:
gen_settings = ExLlamaV2Sampler.Settings()
# Prepare sequences
if not isinstance(input_ids, list):