diff --git a/toolkit/stable_diffusion_model.py b/toolkit/stable_diffusion_model.py
index a5c2eefe..0528b4a6 100644
--- a/toolkit/stable_diffusion_model.py
+++ b/toolkit/stable_diffusion_model.py
@@ -557,6 +557,7 @@ class StableDiffusion:
                     )
                     pipe.load_lora_weights(double_transformer_lora, adapter_name=f"lora1_double")
                     pipe.fuse_lora()
+                    pipe.unload_lora_weights()
                     transformer.transformer_blocks = transformer.transformer_blocks.to(
                         'cpu', dtype=dtype
                     )
@@ -567,6 +568,7 @@ class StableDiffusion:
                     )
                     pipe.load_lora_weights(single_transformer_lora, adapter_name=f"lora1_single")
                     pipe.fuse_lora()
+                    pipe.unload_lora_weights()
                     transformer.single_transformer_blocks = transformer.single_transformer_blocks.to(
                         'cpu', dtype=dtype
                     )