ostris/ai-toolkit
1
0
Fork 0
mirror of https://github.com/ostris/ai-toolkit.git synced 2026-01-26 16:39:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix issue where text encoder could be the wrong quantization and fail when using memory manager

Browse Source
This commit is contained in:
Jaret Burkett
2025-10-15 11:01:30 -06:00
parent 7abf5e20be
commit 1f81bc4060
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
toolkit/config_modules.py
View File

@@ -632,7 +632,7 @@ class ModelConfig:
self.layer_offloading = kwargs.get("layer_offloading", self.auto_memory )
if self.layer_offloading and self.qtype == "qfloat8":
self.qtype = "float8"
if self.layer_offloading and not self.qtype_te == "qfloat8":
if self.layer_offloading and self.qtype_te == "qfloat8":
self.qtype_te = "float8"
# 0 is off and 1.0 is 100% of the layers