theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-30 03:01:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

API: Persist request IDs and append full_text to finish chunk

Browse Source 
Adding these to each generation chunk helps remove redundancy and
unecessary request ID operations.

Signed-off-by: kingbri <8082010+kingbri1@users.noreply.github.com>
This commit is contained in:
kingbri
2025-07-24 22:28:40 -04:00
parent e77fa0b7a8
commit 0b4ca567f8
3 changed files with 19 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
backends/exllamav2/model.py
View File

@@ -909,7 +909,9 @@ class ExllamaV2Container(BaseModelContainer):
generations.append(generation)
joined_generation = {
"request_id": "",
"text": "",
"full_text": "",
"prompt_tokens": 0,
"gen_tokens": 0,
"tool_calls": None,
@@ -923,12 +925,12 @@ class ExllamaV2Container(BaseModelContainer):
if "finish_reason" in generations[-1]:
finish_chunk = generations.pop()
joined_generation = {**joined_generation, **finish_chunk}
joined_generation["text"] = joined_generation.get("full_text", "")
else:
joined_generation["finish_reason"] = "stop"
if len(generations) > 0:
for generation in generations:
joined_generation["text"] += unwrap(generation.get("text"), "")
joined_generation["offset"].append(unwrap(generation.get("offset"), -1))
joined_generation["token_probs"].update(
unwrap(generation.get("token_probs"), {})
@@ -1170,7 +1172,7 @@ class ExllamaV2Container(BaseModelContainer):
}
# Creates and returns a finish chunk
def handle_finish_chunk(self, result: dict, generation: dict):
def handle_finish_chunk(self, result: dict, request_id: str, full_text: str):
eos_reason = result.get("eos_reason")
stop_str = None
@@ -1204,6 +1206,7 @@ class ExllamaV2Container(BaseModelContainer):
total_time = round(queue_time + prompt_time + gen_time, 2)
finish_chunk = {
"request_id": request_id,
"prompt_tokens": prompt_tokens,
"prompt_time": round(prompt_time, 2),
"prompt_tokens_per_sec": prompt_ts,
@@ -1215,6 +1218,7 @@ class ExllamaV2Container(BaseModelContainer):
"cached_tokens": cached_tokens,
"finish_reason": finish_reason,
"stop_str": stop_str,
"full_text": full_text,
}
return finish_chunk
@@ -1414,6 +1418,7 @@ class ExllamaV2Container(BaseModelContainer):
generated_tokens += chunk_tokens.size(dim=0)
generation = {
"request_id": request_id,
"text": chunk,
"prompt_tokens": context_len,
"generated_tokens": generated_tokens,
@@ -1434,7 +1439,9 @@ class ExllamaV2Container(BaseModelContainer):
if result.get("eos"):
log_response(request_id, full_response)
finish_chunk = self.handle_finish_chunk(result, generation)
finish_chunk = self.handle_finish_chunk(
result, request_id, full_response
)
# Save the final result for metrics logging
metrics_result = finish_chunk