turboderp-org/exllamav2
1
0
Fork 0
mirror of https://github.com/turboderp-org/exllamav2.git synced 2026-05-11 08:20:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix 2-byte UTF8 decoding

Browse Source
This commit is contained in:
turboderp
2023-11-10 21:00:29 +01:00
parent b302e310c8
commit 7785bd4efe
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
exllamav2/generator/streaming.py
View File

@@ -220,6 +220,7 @@ class ExLlamaV2StreamingGenerator(ExLlamaV2BaseGenerator):
b = id_to_ord[t]
if 0 < b < 256:
if b & 0b1100000 == 0b1000000: self.expect_utf8 = 2
if b & 0b1110000 == 0b1100000: self.expect_utf8 = 3
if b & 0b1111000 == 0b1110000: self.expect_utf8 = 4
if b & 0b1111100 == 0b1111000: self.expect_utf8 = 5