theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-03-14 15:57:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11 from veden/patch-1

Browse Source 
Fix incorrect ratio calculation for draft model
This commit is contained in:
turboderp
2023-11-20 04:23:34 +01:00
committed by GitHub
parent 4cddd0400c f960fac8ff
commit 8ef730f016
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
model.py
View File

@@ -101,8 +101,6 @@ class ModelContainer:
self.draft_config.model_dir = str(draft_model_path.resolve())
self.draft_config.prepare()
self.draft_config.max_seq_len = self.config.max_seq_len
if "draft_rope_alpha" in kwargs:
self.draft_config.scale_alpha_value = kwargs.get("draft_rope_alpha") or 1
else:
@@ -110,6 +108,8 @@ class ModelContainer:
alpha = -0.13436 + 0.80541 * ratio + 0.28833 * ratio ** 2
self.draft_config.scale_alpha_value = alpha
self.draft_config.max_seq_len = self.config.max_seq_len
if "chunk_size" in kwargs:
self.draft_config.max_input_len = kwargs["chunk_size"]
self.draft_config.max_attn_size = kwargs["chunk_size"] ** 2