mirror of
https://github.com/theroyallab/tabbyAPI.git
synced 2026-05-11 16:30:16 +00:00
79d581e1f5
- remove disconnect_task - move disconnect logic to a per-request handler that wraps cleanup operation and directly polls the request state with throttling - exclusively signal disconnect with CancelledError - rework completions endpoint to follow same approach as chat completions, share some code - refactor OAI endpoints a bit - correct behavior for batched completion requests - make sure logprobs work for completion and streaming completion requests - more tests
48 lines
1.2 KiB
Python
48 lines
1.2 KiB
Python
import random
|
|
import string
|
|
|
|
import yaml
|
|
from _common import *
|
|
|
|
BASE_URL = "http://localhost:5000/v1"
|
|
MODEL = "/mnt/str/models/qwen3.5-35b-a3b/exl3/4.09bpw/"
|
|
|
|
oai_request = {
|
|
"model": MODEL,
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": "".join(random.choices(string.ascii_letters, k=4))
|
|
+ "All work and no play. " * 5000,
|
|
}
|
|
],
|
|
"max_tokens": 500,
|
|
"stream_options": {"include_usage": True},
|
|
}
|
|
|
|
non_tool_request = {
|
|
"model": MODEL,
|
|
"template_vars": {
|
|
"enable_thinking": False,
|
|
},
|
|
"messages": [{"role": "user", "content": "Hello."}],
|
|
}
|
|
|
|
|
|
def main():
|
|
with open("api_tokens.yml") as f:
|
|
tokens = yaml.safe_load(f)
|
|
api_key = tokens["admin_key"]
|
|
|
|
test_chat_streaming(api_key, BASE_URL, non_tool_request.copy(), n=1)
|
|
test_chat_request(api_key, BASE_URL, oai_request.copy(), n=1)
|
|
test_chat_request(api_key, BASE_URL, oai_request.copy(), n=2)
|
|
test_chat_request(api_key, BASE_URL, oai_request.copy(), n=4)
|
|
test_chat_streaming(api_key, BASE_URL, oai_request.copy(), n=1)
|
|
test_chat_streaming(api_key, BASE_URL, oai_request.copy(), n=2)
|
|
test_chat_streaming(api_key, BASE_URL, oai_request.copy(), n=4)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|