theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-29 02:31:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Model: Fix logit bias handling

Browse Source 
If the token doesn't exist, gracefully warn instead of erroring out.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri
2024-02-18 18:30:58 -05:00
parent aa34b2e5fd
commit 7def32e4de
2 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backends/exllamav2/model.py
View File

@@ -750,7 +750,13 @@ class ExllamaV2Container:
# Map logits to the tensor with their biases # Map logits to the tensor with their biases
for token, bias in logit_bias.items(): for token, bias in logit_bias.items():
gen_settings.token_bias[token] = bias if token in gen_settings.token_bias:
gen_settings.token_bias[token] = bias
else:
logger.warning(
f"Logit bias: Token {token} not present "
"in the model's vocab. Skipping."
)
# Ban the EOS token if specified. If not, append to stop conditions # Ban the EOS token if specified. If not, append to stop conditions
# as well. # as well.

2
common/sampling.py
View File

@@ -111,7 +111,7 @@ class BaseSamplerRequest(BaseModel):
logit_bias: Optional[Dict[int, float]] = Field( logit_bias: Optional[Dict[int, float]] = Field(
default_factory=lambda: get_default_sampler_value("logit_bias"), default_factory=lambda: get_default_sampler_value("logit_bias"),
examples=[[{"1": 10}]], examples=[{"1": 10, "2": 50}],
) )
negative_prompt: Optional[str] = Field( negative_prompt: Optional[str] = Field(