From 61f6e51fdbdd2b56ef122d3f4004cd65ec3e90fa Mon Sep 17 00:00:00 2001 From: kingbri Date: Fri, 1 Dec 2023 23:30:19 -0500 Subject: [PATCH] OAI: Add separator style fallback Some models may return None for separator style with FastChat. Fall back to LLAMA2 if this is the case. Signed-off-by: kingbri --- OAI/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/OAI/utils.py b/OAI/utils.py index 7fc3126..a60e3d9 100644 --- a/OAI/utils.py +++ b/OAI/utils.py @@ -16,6 +16,7 @@ from typing import Optional, List try: import fastchat from fastchat.model.model_adapter import get_conversation_template + from fastchat.conversation import SeparatorStyle _fastchat_available = True except ImportError: _fastchat_available = False @@ -116,6 +117,9 @@ def get_chat_completion_prompt(model_path: str, messages: List[ChatCompletionMes ) conv = get_conversation_template(model_path) + if conv.sep_style is None: + conv.sep_style = SeparatorStyle.LLAMA2 + for message in messages: msg_role = message.role if msg_role == "system":