ostris/ai-toolkit
1
0
Fork 0
mirror of https://github.com/ostris/ai-toolkit.git synced 2026-04-30 19:21:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added base for using guidance during training. Still not working right.

Browse Source
This commit is contained in:
Jaret Burkett
2023-11-05 04:03:32 -07:00
parent d35733ac06
commit 8a9e8f708f
5 changed files with 245 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
jobs/process/BaseSDTrainProcess.py
View File

@@ -574,7 +574,12 @@ class BaseSDTrainProcess(BaseTrainProcess):
else:
latents = self.sd.encode_images(imgs)
batch.latents = latents
# flush() # todo check performance removing this
if batch.unconditional_tensor is not None and batch.unconditional_latents is None:
unconditional_imgs = batch.unconditional_tensor
unconditional_imgs = unconditional_imgs.to(self.device_torch, dtype=dtype)
unconditional_latents = self.sd.encode_images(unconditional_imgs)
batch.unconditional_latents = unconditional_latents
unaugmented_latents = None
if self.train_config.loss_target == 'differential_noise':
@@ -655,6 +660,10 @@ class BaseSDTrainProcess(BaseTrainProcess):
noisy_latents = self.sd.noise_scheduler.add_noise(latents, noise, timesteps)
# determine scaled noise
# todo do we need to scale this or does it always predict full intensity
# noise = noisy_latents - latents
# https://github.com/huggingface/diffusers/blob/324d18fba23f6c9d7475b0ff7c777685f7128d40/examples/t2i_adapter/train_t2i_adapter_sdxl.py#L1170C17-L1171C77
if self.train_config.loss_target == 'source' or self.train_config.loss_target == 'unaugmented':
sigmas = self.get_sigmas(timesteps, len(noisy_latents.shape), noisy_latents.dtype)