diff --git a/backends/exllamav2/model.py b/backends/exllamav2/model.py index 1bf1d53..7608a55 100644 --- a/backends/exllamav2/model.py +++ b/backends/exllamav2/model.py @@ -750,7 +750,13 @@ class ExllamaV2Container: # Map logits to the tensor with their biases for token, bias in logit_bias.items(): - gen_settings.token_bias[token] = bias + if token in gen_settings.token_bias: + gen_settings.token_bias[token] = bias + else: + logger.warning( + f"Logit bias: Token {token} not present " + "in the model's vocab. Skipping." + ) # Ban the EOS token if specified. If not, append to stop conditions # as well. diff --git a/common/sampling.py b/common/sampling.py index 3d0ce32..9b78824 100644 --- a/common/sampling.py +++ b/common/sampling.py @@ -111,7 +111,7 @@ class BaseSamplerRequest(BaseModel): logit_bias: Optional[Dict[int, float]] = Field( default_factory=lambda: get_default_sampler_value("logit_bias"), - examples=[[{"1": 10}]], + examples=[{"1": 10, "2": 50}], ) negative_prompt: Optional[str] = Field(