Templates: Fix stop_string parsing

Template modules grab all set vars, including ones that use runtime
vars. If a template var is set to a runtime var and a module is created,
an UndefinedError fires.

Use make_module instead to pass runtime vars when creating a template
module.

Resolves #92

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri
2024-04-02 00:44:04 -04:00
parent 6ecce1604b
commit f9f8c97c6d
2 changed files with 22 additions and 28 deletions

View File

@@ -141,11 +141,14 @@ def format_prompt_with_template(data: ChatCompletionRequest):
unwrap(data.ban_eos_token, False),
)
template_vars = {
"messages": data.messages,
"add_generation_prompt": data.add_generation_prompt,
**special_tokens_dict,
}
prompt, template_stop_strings = get_prompt_from_template(
data.messages,
model.container.prompt_template,
data.add_generation_prompt,
special_tokens_dict,
model.container.prompt_template, template_vars
)
# Append template stop strings
@@ -157,17 +160,17 @@ def format_prompt_with_template(data: ChatCompletionRequest):
return prompt
except KeyError as exc:
raise HTTPException(
400,
error_message = handle_request_error(
"Could not find a Conversation from prompt template "
f"'{model.container.prompt_template.name}'. "
"Check your spelling?",
) from exc
).error.message
raise HTTPException(400, error_message) from exc
except TemplateError as exc:
raise HTTPException(
400,
f"TemplateError: {str(exc)}",
) from exc
error_message = handle_request_error(f"TemplateError: {str(exc)}").error.message
raise HTTPException(400, error_message) from exc
async def stream_generate_chat_completion(