theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-03-14 15:57:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix incorrect ratio calculation for draft model

Browse Source
This commit is contained in:
Veden
2023-11-19 13:12:53 -08:00
committed by GitHub
parent 4cddd0400c
commit f960fac8ff
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
model.py
View File

@@ -101,8 +101,6 @@ class ModelContainer:
self.draft_config.model_dir = str(draft_model_path.resolve()) self.draft_config.model_dir = str(draft_model_path.resolve())
self.draft_config.prepare() self.draft_config.prepare()
self.draft_config.max_seq_len = self.config.max_seq_len
if "draft_rope_alpha" in kwargs: if "draft_rope_alpha" in kwargs:
self.draft_config.scale_alpha_value = kwargs.get("draft_rope_alpha") or 1 self.draft_config.scale_alpha_value = kwargs.get("draft_rope_alpha") or 1
else: else:
@@ -110,6 +108,8 @@ class ModelContainer:
alpha = -0.13436 + 0.80541 * ratio + 0.28833 * ratio ** 2 alpha = -0.13436 + 0.80541 * ratio + 0.28833 * ratio ** 2
self.draft_config.scale_alpha_value = alpha self.draft_config.scale_alpha_value = alpha
self.draft_config.max_seq_len = self.config.max_seq_len
if "chunk_size" in kwargs: if "chunk_size" in kwargs:
self.draft_config.max_input_len = kwargs["chunk_size"] self.draft_config.max_input_len = kwargs["chunk_size"]
self.draft_config.max_attn_size = kwargs["chunk_size"] ** 2 self.draft_config.max_attn_size = kwargs["chunk_size"] ** 2