From cc2516790d87e74fa0cc27154b66752389951769 Mon Sep 17 00:00:00 2001 From: kingbri Date: Mon, 11 Nov 2024 12:09:27 -0500 Subject: [PATCH] Model: Add support for chat_template.json HuggingFace separated the chat template in the newest transformers versions. Signed-off-by: kingbri --- backends/exllamav2/model.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backends/exllamav2/model.py b/backends/exllamav2/model.py index 6f17570..c7d2069 100644 --- a/backends/exllamav2/model.py +++ b/backends/exllamav2/model.py @@ -389,6 +389,10 @@ class ExllamaV2Container: logger.info("Attempting to load a prompt template if present.") find_template_functions = [ + lambda: PromptTemplate.from_model_json( + pathlib.Path(self.config.model_dir) / "chat_template.json", + key="chat_template", + ), lambda: PromptTemplate.from_model_json( pathlib.Path(self.config.model_dir) / "tokenizer_config.json", key="chat_template",