diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py
index 2267db40..73335083 100644
--- a/modules/sd_samplers_common.py
+++ b/modules/sd_samplers_common.py
@@ -183,16 +183,15 @@ def apply_refiner(cfg_denoiser, x):
 
     refiner = sd_models.model_data.get_sd_model()
 
+    devices.torch_gc()
+    cfg_denoiser.p.setup_conds()
+    cfg_denoiser.update_inner_model()
+
     inference_memory = 0
     unet_patcher = refiner.unet_patcher
     model_management.load_models_gpu(
         [unet_patcher],
         unet_patcher.memory_required([x.shape[0]] + list(x.shape[1:])) + inference_memory)
-
-    devices.torch_gc()
-    cfg_denoiser.p.setup_conds()
-    cfg_denoiser.update_inner_model()
-
     return True