theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-28 18:21:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #390

Browse Source
This commit is contained in:
turboderp
2025-10-15 10:40:19 +02:00
parent ad64942fa1
commit 0af29d957a
2 changed files with 8 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
backends/exllamav2/model.py
View File

@@ -1336,14 +1336,9 @@ class ExllamaV2Container(BaseModelContainer):
negative_context_len = input_ids[1].size(dim=-1) if negative_prompt else 0
# Automatically set max_tokens to fill up the context
# This should be an OK default, but may be changed in the future
max_tokens = unwrap(
params.max_tokens,
self.config.max_seq_len - max(context_len, negative_context_len),
)
if max_tokens < 1:
logger.warning("max_tokens must be a positive integer, setting to 1.")
max_tokens = 1
max_tokens = unwrap(params.max_tokens, 0)
if max_tokens <= 0:
max_tokens = self.config.max_seq_len - max(context_len, negative_context_len)
# Determine if the negative context or the context length is bigger
context_to_check = max(negative_context_len, context_len)