From 7e669527ed9213ffd8cf67332db7b15e4b3b9129 Mon Sep 17 00:00:00 2001 From: kingbri Date: Wed, 20 Mar 2024 11:23:17 -0400 Subject: [PATCH] Model: Fix tokenizer bugs Some tokenizer variables don't get cleaned up on init, so these can persist. Clean these up manually before creating a new tokenizer for now. Signed-off-by: kingbri --- backends/exllamav2/model.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backends/exllamav2/model.py b/backends/exllamav2/model.py index c169b9c..c008b8d 100644 --- a/backends/exllamav2/model.py +++ b/backends/exllamav2/model.py @@ -390,7 +390,12 @@ class ExllamaV2Container: # Notify that the model is being loaded self.model_is_loading = True - # Load tokenizer + # Reset tokenizer namespace vars and create a tokenizer + ExLlamaV2Tokenizer.unspecial_piece_to_id = {} + ExLlamaV2Tokenizer.unspecial_id_to_piece = {} + ExLlamaV2Tokenizer.extended_id_to_piece = {} + ExLlamaV2Tokenizer.extended_piece_to_id = {} + self.tokenizer = ExLlamaV2Tokenizer(self.config) # Calculate autosplit reserve for all GPUs