theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-27 09:41:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Config: Add option to force streaming off

Browse Source 
Many APIs automatically ask for request streaming without giving
the user the option to turn it off. Therefore, give the user more
freedom by giving a server-side kill switch.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri
2024-02-07 21:08:21 -05:00
parent d0027bce32
commit 58590a6c57
3 changed files with 19 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
main.py
View File

@@ -449,7 +449,11 @@ async def generate_completion(request: Request, data: CompletionRequest):
if isinstance(data.prompt, list):
data.prompt = "\n".join(data.prompt)
if data.stream:
disable_request_streaming = unwrap(
get_developer_config().get("disable_request_streaming"), False
)
if data.stream and not disable_request_streaming:
async def generator():
"""Generator for the generation process."""
@@ -531,7 +535,11 @@ async def generate_chat_completion(request: Request, data: ChatCompletionRequest
f"TemplateError: {str(exc)}",
) from exc
if data.stream:
disable_request_streaming = unwrap(
get_developer_config().get("disable_request_streaming"), False
)
if data.stream and not disable_request_streaming:
const_id = f"chatcmpl-{uuid4().hex}"
async def generator():