diff --git a/toolkit/guidance.py b/toolkit/guidance.py
index 287d17e7..340b22e0 100644
--- a/toolkit/guidance.py
+++ b/toolkit/guidance.py
@@ -453,12 +453,14 @@ def get_guided_loss_polarity(
             noise,
             timesteps
         ).detach()
+        conditional_noisy_latents = sd.condition_noisy_latents(conditional_noisy_latents, batch)
 
         unconditional_noisy_latents = sd.add_noise(
             unconditional_latents,
             noise,
             timesteps
         ).detach()
+        unconditional_noisy_latents = sd.condition_noisy_latents(unconditional_noisy_latents, batch)
 
         # double up everything to run it through all at once
         cat_embeds = concat_prompt_embeds([conditional_embeds, conditional_embeds])