diff --git a/model.py b/model.py index 0bcc87d..8d3480b 100644 --- a/model.py +++ b/model.py @@ -270,6 +270,23 @@ class ModelContainer: # Sampler settings gen_settings = ExLlamaV2Sampler.Settings() + + # Warn if unsupported settings supplied + + if "mirostat" in kwargs and not hasattr(gen_settings, "mirostat"): + print(" !! Warning: Currently installed ExLlamaV2 does not support Mirostat sampling") + + if kwargs.get("min_p", 0.0) not in [0.0, 1.0] and not hasattr(gen_settings, "min_p"): + print(" !! Warning: Currently installed ExLlamaV2 does not support min-P sampling") + + if kwargs.get("tfs", 0.0) not in [0.0, 1.0] and not hasattr(gen_settings, "tfs"): + print(" !! Warning: Currently installed ExLlamaV2 does not support tail-free sampling (TFS)") + + if "temperature_last" in kwargs and not hasattr(gen_settings, "temperature_last"): + print(" !! Warning: Currently installed ExLlamaV2 does not support temperature_last") + + #Apply settings + gen_settings.temperature = kwargs.get("temperature", 1.0) gen_settings.temperature_last = kwargs.get("temperature_last", False) gen_settings.top_k = kwargs.get("top_k", 1)