mirror of
https://github.com/theroyallab/tabbyAPI.git
synced 2026-03-14 15:57:27 +00:00
Templates: Support bos_token and eos_token fields
These are commonly seen in huggingface provided chat templates and aren't that difficult to add in. For feature parity, honor the add_bos_token and ban_eos_token parameters when constructing the prompt. Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
7
main.py
7
main.py
@@ -324,10 +324,15 @@ async def generate_chat_completion(request: Request, data: ChatCompletionRequest
|
||||
prompt = data.messages
|
||||
else:
|
||||
try:
|
||||
special_tokens_dict = model_container.get_special_tokens(
|
||||
unwrap(data.add_bos_token, True),
|
||||
unwrap(data.ban_eos_token, False)
|
||||
)
|
||||
prompt = get_prompt_from_template(
|
||||
data.messages,
|
||||
model_container.prompt_template,
|
||||
data.add_generation_prompt
|
||||
data.add_generation_prompt,
|
||||
special_tokens_dict,
|
||||
)
|
||||
except KeyError:
|
||||
return HTTPException(
|
||||
|
||||
Reference in New Issue
Block a user