API: Add logprobs for chat completions

Adds chat completion logprob support using OAI's spec. Tokens are
not converted to tiktoken here since that will add an extra dependency
for no real reason.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri
2024-02-08 01:16:38 -05:00
committed by Brian Dashore
parent c02fe4d1db
commit c7428f0bcd
3 changed files with 58 additions and 8 deletions

View File

@@ -520,6 +520,8 @@ class ExllamaV2Container:
joined_generation["token_probs"].update(
unwrap(generation.get("token_probs"), {})
)
# Include empty logprob dicts for index preservation
joined_generation["logprobs"].append(
unwrap(generation.get("logprobs"), {})
)