Merge pull request #112 from DocShotgun/main

Separate new prompt tokens from those reused from cache in metric logging
2026-04-28 02:01:24 +00:00 · 2024-05-27 18:04:43 -04:00
parent 19961f4126 7084081b1f
commit 516b52b341
2 changed files with 9 additions and 2 deletions
--- a/backends/exllamav2/model.py
+++ b/backends/exllamav2/model.py
@@ -1144,6 +1144,7 @@ class ExllamaV2Container:
                        log_metrics(
                            result.get("time_enqueued"),
                            result.get("prompt_tokens"),
+                            result.get("cached_tokens"),
                            result.get("time_prefill"),
                            result.get("new_tokens"),
                            result.get("time_generate"),