From 7785bd4efe6cac0208eb2f9d374a95958ce31d58 Mon Sep 17 00:00:00 2001 From: turboderp Date: Fri, 10 Nov 2023 21:00:29 +0100 Subject: [PATCH] Fix 2-byte UTF8 decoding --- exllamav2/generator/streaming.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exllamav2/generator/streaming.py b/exllamav2/generator/streaming.py index b45107c..9810e45 100644 --- a/exllamav2/generator/streaming.py +++ b/exllamav2/generator/streaming.py @@ -220,6 +220,7 @@ class ExLlamaV2StreamingGenerator(ExLlamaV2BaseGenerator): b = id_to_ord[t] if 0 < b < 256: + if b & 0b1100000 == 0b1000000: self.expect_utf8 = 2 if b & 0b1110000 == 0b1100000: self.expect_utf8 = 3 if b & 0b1111000 == 0b1110000: self.expect_utf8 = 4 if b & 0b1111100 == 0b1111000: self.expect_utf8 = 5