diff --git a/toolkit/config_modules.py b/toolkit/config_modules.py index 1fa7c688..a6c89576 100644 --- a/toolkit/config_modules.py +++ b/toolkit/config_modules.py @@ -632,7 +632,7 @@ class ModelConfig: self.layer_offloading = kwargs.get("layer_offloading", self.auto_memory ) if self.layer_offloading and self.qtype == "qfloat8": self.qtype = "float8" - if self.layer_offloading and not self.qtype_te == "qfloat8": + if self.layer_offloading and self.qtype_te == "qfloat8": self.qtype_te = "float8" # 0 is off and 1.0 is 100% of the layers