From 31bc4187955b7a62cd180f6cad65530e63f14ca4 Mon Sep 17 00:00:00 2001 From: kingbri Date: Sun, 19 Nov 2023 00:49:32 -0500 Subject: [PATCH] Model: Add context in response output When printing to the console, give information about the context (ingestion token count). Signed-off-by: kingbri --- model.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/model.py b/model.py index 7274fa0..ed8d50f 100644 --- a/model.py +++ b/model.py @@ -373,7 +373,10 @@ class ModelContainer: # Add tokens per second extra_responses.append(f"{'Indeterminate' if elapsed_time == 0 else round(generated_tokens / elapsed_time, 2)} T/s") - extra_responses.append(f"{generated_tokens} tokens") + + # Add context (original token count) + if ids is not None: + extra_responses.append(f"context {len(ids[0])} tokens") # Print output print(initial_response + " (" + ", ".join(extra_responses) + ")")