theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-03-14 15:57:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Model: Set default max_batch_size

Browse Source
This commit is contained in:
turboderp
2025-05-03 18:32:30 +02:00
parent 8c75b29923
commit 0d949d00b9
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backends/exllamav3/model.py
View File

@@ -180,7 +180,7 @@ class ExllamaV3Container(BaseModelContainer):
self.cache = Cache(self.model, max_num_tokens=self.cache_size)
# Max batch size
self.max_batch_size = kwargs.get("max_batch_size")
self.max_batch_size = unwrap(kwargs.get("max_batch_size"), 256)
# Make sure chunk size is >= 256, keep near or below max seq len
user_chunk_size = unwrap(kwargs.get("chunk_size"), 2048)