theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-27 17:51:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

OAI: Add stream to gen params

Browse Source 
Good for logging.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri
2024-03-21 00:55:44 -04:00
parent 69e41e994c
commit 56fdfb5f8e
2 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backends/exllamav2/model.py
View File

@@ -863,6 +863,7 @@ class ExllamaV2Container:
# Some options are too large, so log the args instead # Some options are too large, so log the args instead
log_generation_params( log_generation_params(
max_tokens=max_tokens, max_tokens=max_tokens,
stream=kwargs.get("stream"),
**gen_settings_log_dict, **gen_settings_log_dict,
token_healing=token_healing, token_healing=token_healing,
auto_scale_penalty_range=auto_scale_penalty_range, auto_scale_penalty_range=auto_scale_penalty_range,

5
endpoints/OAI/types/common.py
View File

@@ -43,6 +43,9 @@ class CommonCompletionRequest(BaseSamplerRequest):
) )
def to_gen_params(self): def to_gen_params(self):
extra_gen_params = {"logprobs": self.logprobs} extra_gen_params = {
"stream": self.stream,
"logprobs": self.logprobs,
}
return super().to_gen_params(**extra_gen_params) return super().to_gen_params(**extra_gen_params)