theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-03-14 15:57:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #253 from randoentity/workaround-toolcall

Browse Source 
workaround for tool calling
This commit is contained in:
Brian
2024-11-28 23:30:00 -05:00
committed by GitHub
parent b81dcdaf66 2e06fb01d3
commit fe44e4a524
1 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
endpoints/OAI/utils/chat_completion.py
View File

@@ -454,16 +454,23 @@ async def generate_tool_calls(
if gen["stop_str"] in tool_data.tool_call_start:
if "text" in gen:
# non streaming, all generations will have the text they generated
pre_tool_prompt = await apply_chat_template(data, gen["text"])
pre_tool_prompt, mm_embeddings = await apply_chat_template(
data, gen["text"]
)
elif current_generations is not None:
# streaming, we wont have text in the generation,
# we'll have to use the current_generations
pre_tool_prompt = await apply_chat_template(data, current_generations)
pre_tool_prompt, mm_embeddings = await apply_chat_template(
data, current_generations
)
gen_tasks.append(
asyncio.create_task(
model.container.generate(
pre_tool_prompt, request.state.id, **gen_params
pre_tool_prompt,
request.state.id,
embeddings=mm_embeddings,
**gen_params,
)
)
)