Merge pull request #21 from SinanAkkoyun/speculative-decoding-fix

Set ropescale of draft model

.gitignore

@@ -0,0 +1,8 @@
+*.egg-info/
+build/
+
+*.pyc
+__pycache__/
+.idea
+venv
+dist

backend/models.py

@@ -228,6 +228,7 @@ class ModelContainer:
             self.draft_config.max_seq_len = self.config.max_seq_len
 
             self.draft_config.scale_alpha_value = alpha
+            self.draft_config.scale_pos_emb = model["rope_scale"]
             self.draft_config.max_input_len = model["chunk_size"]
             self.draft_config.max_attn_size = model["chunk_size"] ** 2