Dependencies: Pin tokenizers

Use a version greater than 0.20.0 for newer model support. Signed-off-by: kingbri <8082010+bdashore3@users.noreply.github.com>
2026-03-15 00:07:28 +00:00 · 2024-12-13 00:58:25 -05:00
parent 1ba33bf646
commit bc3c154c96
1 changed files with 1 additions and 1 deletions
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,7 +25,7 @@ dependencies = [
    "loguru",
    "sse-starlette",
    "packaging",
-    "tokenizers",
+    "tokenizers>=0.21.0",
    "formatron",
    "kbnf>=0.4.1",
    "aiofiles",