theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-04-19 22:08:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixup: max_seq_len

Browse Source 
Whoops
This commit is contained in:
randoentity
2025-04-30 00:23:25 +02:00
committed by kingbri
parent daae9ec43d
commit 14fb573371
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backends/exllamav3/model.py
View File

@@ -47,7 +47,7 @@ class ExllamaV3Container(BaseModelContainer):
tokenizer: Tokenizer
config: Config
gpu_split: List[float] = []
max_seq_len: int = 2048
max_seq_len: int
# Required methods
@classmethod
@@ -74,8 +74,8 @@ class ExllamaV3Container(BaseModelContainer):
self.model = Model.from_config(self.config)
self.tokenizer = Tokenizer.from_config(self.config)
max_seq_len = kwargs.get("max_seq_len")
self.cache = Cache(self.model, max_num_tokens=max_seq_len)
self.max_seq_len = kwargs.get("max_seq_len")
self.cache = Cache(self.model, max_num_tokens=self.max_seq_len)
gpu_split = unwrap(kwargs.get("gpu_split"), [])
# Set GPU split options