theroyallab/tabbyAPI
1
0
Fork 0
mirror of https://github.com/theroyallab/tabbyAPI.git synced 2026-03-14 15:57:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Force auto-alpha to 1.0 if config ctx == base ctx

Browse Source
This commit is contained in:
DocShotgun
2023-12-02 21:19:59 -08:00
committed by GitHub
parent 1c398b0be7
commit bd2c5d0d09
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
model.py
View File

@@ -79,6 +79,7 @@ class ModelContainer:
else:
ratio = self.config.max_seq_len / base_seq_len
alpha = -0.13436 + 0.80541 * ratio + 0.28833 * ratio ** 2
if ratio == 1: alpha = 1.0
self.config.scale_alpha_value = alpha
if "no_flash_attn" in kwargs: self.config.no_flash_attn = kwargs["no_flash_attn"]
@@ -113,6 +114,7 @@ class ModelContainer:
else:
ratio = self.config.max_seq_len / self.draft_config.max_seq_len
alpha = -0.13436 + 0.80541 * ratio + 0.28833 * ratio ** 2
if ratio == 1: alpha = 1.0
self.draft_config.scale_alpha_value = alpha
self.draft_config.max_seq_len = self.config.max_seq_len