From a8481c1670eb890a9ec9ffc2bfdecee1bc5eed69 Mon Sep 17 00:00:00 2001 From: Jaret Burkett Date: Tue, 6 Feb 2024 03:44:54 -0700 Subject: [PATCH] randomly adjust scale of unconditional noise on ip adapters if training with cfg --- toolkit/ip_adapter.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/toolkit/ip_adapter.py b/toolkit/ip_adapter.py index a08d523f..3a31c8b0 100644 --- a/toolkit/ip_adapter.py +++ b/toolkit/ip_adapter.py @@ -500,6 +500,9 @@ class IPAdapter(torch.nn.Module): if drop: if self.clip_noise_zero: tensors_0_1 = torch.rand_like(tensors_0_1).detach() + noise_scale = torch.rand([tensors_0_1.shape[0], 1, 1, 1], device=self.device, + dtype=get_torch_dtype(self.sd_ref().dtype)) + tensors_0_1 = tensors_0_1 * noise_scale else: tensors_0_1 = torch.zeros_like(tensors_0_1).detach() # tensors_0_1 = tensors_0_1 * 0 @@ -514,6 +517,9 @@ class IPAdapter(torch.nn.Module): # scale the noise down if self.clip_noise_zero: tensors_0_1 = torch.rand_like(tensors_0_1).detach() + noise_scale = torch.rand([tensors_0_1.shape[0], 1, 1, 1], device=self.device, + dtype=get_torch_dtype(self.sd_ref().dtype)) + tensors_0_1 = tensors_0_1 * noise_scale else: tensors_0_1 = torch.zeros_like(tensors_0_1).detach() # tensors_0_1 = tensors_0_1 * 0