turboderp-org/exllamav2
1
0
Fork 0
mirror of https://github.com/turboderp-org/exllamav2.git synced 2026-04-30 03:01:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Generator: Make sampler settings optional instead of default arg

Browse Source
This commit is contained in:
turboderp
2024-11-12 03:41:59 +01:00
parent 90895967b1
commit 16cd5ef384
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
exllamav2/generator/dynamic.py
View File

@@ -655,7 +655,7 @@ class ExLlamaV2DynamicGenerator:
assert False, "Unexpected type in prompt" assert False, "Unexpected type in prompt"
if gen_settings is None: if gen_settings is None:
p_settings = ExLlamaV2Sampler.Settings() p_settings = None
elif isinstance(gen_settings, ExLlamaV2Sampler.Settings): elif isinstance(gen_settings, ExLlamaV2Sampler.Settings):
p_settings = gen_settings p_settings = gen_settings
elif isinstance(gen_settings, list): elif isinstance(gen_settings, list):
@@ -1536,7 +1536,7 @@ class ExLlamaV2DynamicJob:
max_new_tokens: int, max_new_tokens: int,
min_new_tokens: int = 0, min_new_tokens: int = 0,
max_skips: int | None = 4, max_skips: int | None = 4,
gen_settings: ExLlamaV2Sampler.Settings = ExLlamaV2Sampler.Settings(), gen_settings: ExLlamaV2Sampler.Settings | None = None,
seed: int = None, seed: int = None,
stop_conditions: list | tuple | set = None, stop_conditions: list | tuple | set = None,
decode_special_tokens: bool = False, decode_special_tokens: bool = False,
@@ -1629,6 +1629,9 @@ class ExLlamaV2DynamicJob:
self.max_skips = max_skips self.max_skips = max_skips
self.allocated_pages = None self.allocated_pages = None
if gen_settings is None:
gen_settings = ExLlamaV2Sampler.Settings()
# Prepare sequences # Prepare sequences
if not isinstance(input_ids, list): if not isinstance(input_ids, list):