mirror of
https://github.com/kvcache-ai/ktransformers.git
synced 2026-11-05 01:07:28 +00:00
[GH-ISSUE #768] API server error Exception in ASGI application #556
Labels
No milestone
No project
No assignees
1 participant
Notifications
Due date
No due date set.
Dependencies
No dependencies set.
Reference
kvcache-ai/ktransformers#556
Loading…
Add table
Add a link
Reference in a new issue
No description provided.
Delete branch "%!s()"
Deleting a branch is permanent. Although the deleted branch may continue to exist for a short time before it actually gets removed, it CANNOT be undone in most cases. Continue?
Originally created by @3keyallen3 on GitHub (Mar 3, 2025).
Original GitHub issue: https://github.com/kvcache-ai/ktransformers/issues/768
Docker中运行 0.2.2rc1+cu121torch23fancy
ktransformers --gguf_path /models/GGUF --model_path /models --cpu_infer 90 --optimize_config_path /models/DeepSeek-V3-Chat-multi-gpu.yaml --port 10002 --max_new_tokens 1000 --host 0.0.0.0 --model_name DeepSeek-R1
curl -X 'POST' 'http://localhost:10002/api/generate' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{ "model": "DeepSeek-R1", "prompt": "hello.", "stream": true, "temperature": 0.7, "top_p":0.9 }'
curl: (18) transfer closed with outstanding read data remaining
INFO: Started server process [32884]
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Uvicorn running on http://0.0.0.0:10002 (Press CTRL+C to quit)
COMPLETION INPUT:----
hello.
INFO: 172.17.0.1:47250 - "POST /api/generate HTTP/1.1" 200 OK
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/opt/conda/lib/python3.10/site-packages/starlette/responses.py", line 265, in call
await wrap(partial(self.listen_for_disconnect, receive))
File "/opt/conda/lib/python3.10/site-packages/starlette/responses.py", line 261, in wrap
await func()
File "/opt/conda/lib/python3.10/site-packages/starlette/responses.py", line 238, in listen_for_disconnect
message = await receive()
File "/opt/conda/lib/python3.10/site-packages/uvicorn/protocols/http/h11_impl.py", line 534, in receive
await self.message_event.wait()
File "/opt/conda/lib/python3.10/asyncio/locks.py", line 214, in wait
await fut
asyncio.exceptions.CancelledError: Cancelled by cancel scope 7f302c1298d0
During handling of the above exception, another exception occurred:
| File "/opt/conda/lib/python3.10/site-packages/uvicorn/protocols/http/h11_impl.py", line 406, in run_asgi
| result = await app( # type: ignore[func-returns-value]
| File "/opt/conda/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 70, in call
| return await self.app(scope, receive, send)
| File "/opt/conda/lib/python3.10/site-packages/fastapi/applications.py", line 1054, in call
| await super().call(scope, receive, send)
| File "/opt/conda/lib/python3.10/site-packages/starlette/applications.py", line 123, in call
| await self.middleware_stack(scope, receive, send)
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/errors.py", line 186, in call
| raise exc
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/errors.py", line 164, in call
| await self.app(scope, receive, _send)
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/cors.py", line 85, in call
| await self.app(scope, receive, send)
| File "/opt/conda/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 65, in call
| await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
| File "/opt/conda/lib/python3.10/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app
| raise exc
| File "/opt/conda/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
| await app(scope, receive, sender)
| File "/opt/conda/lib/python3.10/site-packages/starlette/routing.py", line 754, in call
| await self.middleware_stack(scope, receive, send)
| File "/opt/conda/lib/python3.10/site-packages/starlette/routing.py", line 774, in app
| await route.handle(scope, receive, send)
| File "/opt/conda/lib/python3.10/site-packages/starlette/routing.py", line 295, in handle
| await self.app(scope, receive, send)
| File "/opt/conda/lib/python3.10/site-packages/starlette/routing.py", line 77, in app
| await wrap_app_handling_exceptions(app, request)(scope, receive, send)
| File "/opt/conda/lib/python3.10/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app
| raise exc
| File "/opt/conda/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
| await app(scope, receive, sender)
| File "/opt/conda/lib/python3.10/site-packages/starlette/routing.py", line 75, in app
| await response(scope, receive, send)
| File "/opt/conda/lib/python3.10/site-packages/starlette/responses.py", line 258, in call
| async with anyio.create_task_group() as task_group:
| File "/opt/conda/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 680, in aexit
| raise BaseExceptionGroup(
| exceptiongroup.ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
+-+---------------- 1 ----------------
| Traceback (most recent call last):
| File "/opt/conda/lib/python3.10/site-packages/starlette/responses.py", line 261, in wrap
| await func()
| File "/opt/conda/lib/python3.10/site-packages/starlette/responses.py", line 250, in stream_response
| async for chunk in self.body_iterator:
| File "/opt/conda/lib/python3.10/site-packages/ktransformers/server/schemas/assistants/streaming.py", line 80, in check_client_link
| async for event in async_events:
| File "/opt/conda/lib/python3.10/site-packages/ktransformers/server/api/ollama/completions.py", line 61, in inner
| async for token in interface.inference(input.prompt, id):
| TypeError: KTransformersInterface.inference() missing 2 required positional arguments: 'temperature' and 'top_p'
+------------------------------------
@qiyuxinlin commented on GitHub (Mar 3, 2025):
We fixed this issue yesterday and need you to pull the latest code to try
@3keyallen3 commented on GitHub (Mar 4, 2025):
@qiyuxinlin 谢谢，更新后，可以使用API了。