From 878c7ecfad9d360c14006fcecd048193b6440c5d Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 28 Jan 2024 07:27:51 -0800 Subject: [PATCH] i --- README.md | 16 ++++++++++++---- .../scripts/sd_forge_controlnet_example.py | 16 +++++++++++++--- modules_forge/controlnet.py | 12 ++++++++++++ 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9b3dc887..0c6138f6 100644 --- a/README.md +++ b/README.md @@ -447,8 +447,8 @@ class ControlNetExampleForge(scripts.Script): unet = p.sd_model.forge_objects.unet - # Unet has input, middle, output blocks, and we can give different - # weights to each layers in all blocks. + # Unet has input, middle, output blocks, and we can give different weights + # to each layers in all blocks. # Below is an example for stronger control in middle block. # This is helpful for some high-res fix passes. (p.is_hr_pass) positive_advanced_weighting = { @@ -465,10 +465,17 @@ class ControlNetExampleForge(scripts.Script): # The advanced_frame_weighting is a weight applied to each image in a batch. # The length of this list must be same with batch size # For example, if batch size is 5, the below list is [0, 0.25, 0.5, 0.75, 1.0] - # If you view the 5 images as 5 frames in a video, this will lead to + # If you view the 5 images as 5 frames in a video, this will lead to # progressively stronger control over time. advanced_frame_weighting = [float(i) / float(batch_size - 1) for i in range(batch_size)] + # The advanced_sigma_weighting allows you to dynamically compute control + # weights given diffusion timestep (sigma). + # For example below code can softly make beginning steps stronger than ending steps. + sigma_max = unet.model.model_sampling.percent_to_sigma(0.0) + sigma_min = unet.model.model_sampling.percent_to_sigma(1.0) + advanced_sigma_weighting = lambda s: (s - sigma_min) / (sigma_max - sigma_min) + # But in this simple example we do not use them positive_advanced_weighting = None negative_advanced_weighting = None @@ -478,7 +485,8 @@ class ControlNetExampleForge(scripts.Script): strength=0.6, start_percent=0.0, end_percent=0.8, positive_advanced_weighting=positive_advanced_weighting, negative_advanced_weighting=negative_advanced_weighting, - advanced_frame_weighting=advanced_frame_weighting) + advanced_frame_weighting=advanced_frame_weighting, + advanced_sigma_weighting=advanced_sigma_weighting) p.sd_model.forge_objects.unet = unet diff --git a/extensions-builtin/sd_forge_controlnet_example/scripts/sd_forge_controlnet_example.py b/extensions-builtin/sd_forge_controlnet_example/scripts/sd_forge_controlnet_example.py index 4bc0cf2a..6c424619 100644 --- a/extensions-builtin/sd_forge_controlnet_example/scripts/sd_forge_controlnet_example.py +++ b/extensions-builtin/sd_forge_controlnet_example/scripts/sd_forge_controlnet_example.py @@ -99,7 +99,8 @@ class ControlNetExampleForge(scripts.Script): unet = p.sd_model.forge_objects.unet - # Unet has input, middle, output blocks, and we can give different weights to each layers in all blocks. + # Unet has input, middle, output blocks, and we can give different weights + # to each layers in all blocks. # Below is an example for stronger control in middle block. # This is helpful for some high-res fix passes. (p.is_hr_pass) positive_advanced_weighting = { @@ -116,9 +117,17 @@ class ControlNetExampleForge(scripts.Script): # The advanced_frame_weighting is a weight applied to each image in a batch. # The length of this list must be same with batch size # For example, if batch size is 5, the below list is [0, 0.25, 0.5, 0.75, 1.0] - # If you view the 5 images as 5 frames in a video, this will lead to progressively stronger control over time. + # If you view the 5 images as 5 frames in a video, this will lead to + # progressively stronger control over time. advanced_frame_weighting = [float(i) / float(batch_size - 1) for i in range(batch_size)] + # The advanced_sigma_weighting allows you to dynamically compute control + # weights given diffusion timestep (sigma). + # For example below code can softly make beginning steps stronger than ending steps. + sigma_max = unet.model.model_sampling.percent_to_sigma(0.0) + sigma_min = unet.model.model_sampling.percent_to_sigma(1.0) + advanced_sigma_weighting = lambda s: (s - sigma_min) / (sigma_max - sigma_min) + # But in this simple example we do not use them positive_advanced_weighting = None negative_advanced_weighting = None @@ -128,7 +137,8 @@ class ControlNetExampleForge(scripts.Script): strength=0.6, start_percent=0.0, end_percent=0.8, positive_advanced_weighting=positive_advanced_weighting, negative_advanced_weighting=negative_advanced_weighting, - advanced_frame_weighting=advanced_frame_weighting) + advanced_frame_weighting=advanced_frame_weighting, + advanced_sigma_weighting=advanced_sigma_weighting) p.sd_model.forge_objects.unet = unet diff --git a/modules_forge/controlnet.py b/modules_forge/controlnet.py index 151e0102..4f58d525 100644 --- a/modules_forge/controlnet.py +++ b/modules_forge/controlnet.py @@ -8,6 +8,7 @@ def apply_controlnet_advanced( positive_advanced_weighting=None, negative_advanced_weighting=None, advanced_frame_weighting=None, + advanced_sigma_weighting=None ): """ @@ -35,12 +36,23 @@ def apply_controlnet_advanced( For example, if batch size is 5, you can use advanced_frame_weighting = [0, 0.25, 0.5, 0.75, 1.0] If you view the 5 images as 5 frames in a video, this will lead to progressively stronger control over time. + # advanced_sigma_weighting + + The advanced_sigma_weighting allows you to dynamically compute control + weights given diffusion timestep (sigma). + For example below code can softly make beginning steps stronger than ending steps. + + sigma_max = unet.model.model_sampling.percent_to_sigma(0.0) + sigma_min = unet.model.model_sampling.percent_to_sigma(1.0) + advanced_sigma_weighting = lambda s: (s - sigma_min) / (sigma_max - sigma_min) + """ cnet = controlnet.copy().set_cond_hint(image_bhwc.movedim(-1, 1), strength, (start_percent, end_percent)) cnet.positive_advanced_weighting = positive_advanced_weighting cnet.negative_advanced_weighting = negative_advanced_weighting cnet.advanced_frame_weighting = advanced_frame_weighting + cnet.advanced_sigma_weighting = advanced_sigma_weighting m = unet.clone() m.add_patched_controlnet(cnet)