theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-24 00:09:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #253 from randoentity/workaround-toolcall

Browse Source 
workaround for tool calling
This commit is contained in:
Brian
2024-11-28 23:30:00 -05:00
committed by GitHub
parent b81dcdaf66 2e06fb01d3
commit fe44e4a524
1 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
endpoints/OAI/utils/chat_completion.py
View File

@@ -454,16 +454,23 @@ async def generate_tool_calls(
if gen["stop_str"] in tool_data.tool_call_start: if gen["stop_str"] in tool_data.tool_call_start:
if "text" in gen: if "text" in gen:
# non streaming, all generations will have the text they generated # non streaming, all generations will have the text they generated
pre_tool_prompt = await apply_chat_template(data, gen["text"]) pre_tool_prompt, mm_embeddings = await apply_chat_template(
data, gen["text"]
)
elif current_generations is not None: elif current_generations is not None:
# streaming, we wont have text in the generation, # streaming, we wont have text in the generation,
# we'll have to use the current_generations # we'll have to use the current_generations
pre_tool_prompt = await apply_chat_template(data, current_generations) pre_tool_prompt, mm_embeddings = await apply_chat_template(
data, current_generations
)
gen_tasks.append( gen_tasks.append(
asyncio.create_task( asyncio.create_task(
model.container.generate( model.container.generate(
pre_tool_prompt, request.state.id, **gen_params pre_tool_prompt,
request.state.id,
embeddings=mm_embeddings,
**gen_params,
) )
) )
) )