From f53fd0869051b9ec6ae02c95f9077f3617da7e3c Mon Sep 17 00:00:00 2001 From: Jaret Burkett Date: Tue, 1 Aug 2023 14:08:22 -0600 Subject: [PATCH] Fixed issue with rescaled loras only saving af fp32 --- jobs/process/ModRescaleLoraProcess.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jobs/process/ModRescaleLoraProcess.py b/jobs/process/ModRescaleLoraProcess.py index 882ef0e0..ff8304d4 100644 --- a/jobs/process/ModRescaleLoraProcess.py +++ b/jobs/process/ModRescaleLoraProcess.py @@ -84,7 +84,8 @@ class ModRescaleLoraProcess(BaseProcess): if self.scale_target == 'up_down' and key.endswith('.lora_up.weight') or key.endswith('.lora_down.weight'): # would it be better to adjust the up weights for fp16 precision? Doing both should reduce chance of NaN v = v * up_down_scale - new_state_dict[key] = v.to(get_torch_dtype(self.save_dtype)) + v = v.detach().clone().to("cpu").to(self.save_dtype) + new_state_dict[key] = v save_meta = add_model_hash_to_meta(new_state_dict, save_meta) save_file(new_state_dict, self.output_path, save_meta)