diff --git a/extensions-builtin/forge_legacy_preprocessors/legacy_preprocessors/preprocessor_compiled.py b/extensions-builtin/forge_legacy_preprocessors/legacy_preprocessors/preprocessor_compiled.py index 746d6896..48cb3150 100644 --- a/extensions-builtin/forge_legacy_preprocessors/legacy_preprocessors/preprocessor_compiled.py +++ b/extensions-builtin/forge_legacy_preprocessors/legacy_preprocessors/preprocessor_compiled.py @@ -860,48 +860,48 @@ legacy_preprocessors = { "Reference" ] }, - "revision_clipvision": { - "label": "revision_clipvision", - "call_function": functools.partial(clip, config='clip_g'), - "unload_function": functools.partial(unload_clip, config='clip_g'), - "managed_model": None, - "model_free": True, - "no_control_mode": True, - "resolution": None, - "slider_1": { - "label": "Noise Augmentation", - "value": 0.0, - "minimum": 0.0, - "maximum": 1.0 - }, - "slider_2": None, - "slider_3": None, - "priority": 100, - "tags": [ - "Revision" - ] - }, - "revision_ignore_prompt": { - "label": "revision_ignore_prompt", - "call_function": functools.partial(clip, config='clip_g'), - "unload_function": functools.partial(unload_clip, config='clip_g'), - "managed_model": None, - "model_free": True, - "no_control_mode": True, - "resolution": None, - "slider_1": { - "label": "Noise Augmentation", - "value": 0.0, - "minimum": 0.0, - "maximum": 1.0 - }, - "slider_2": None, - "slider_3": None, - "priority": 0, - "tags": [ - "Revision" - ] - }, + # "revision_clipvision": { + # "label": "revision_clipvision", + # "call_function": functools.partial(clip, config='clip_g'), + # "unload_function": functools.partial(unload_clip, config='clip_g'), + # "managed_model": None, + # "model_free": True, + # "no_control_mode": True, + # "resolution": None, + # "slider_1": { + # "label": "Noise Augmentation", + # "value": 0.0, + # "minimum": 0.0, + # "maximum": 1.0 + # }, + # "slider_2": None, + # "slider_3": None, + # "priority": 100, + # "tags": [ + # "Revision" + # ] + # }, + # "revision_ignore_prompt": { + # "label": "revision_ignore_prompt", + # "call_function": functools.partial(clip, config='clip_g'), + # "unload_function": functools.partial(unload_clip, config='clip_g'), + # "managed_model": None, + # "model_free": True, + # "no_control_mode": True, + # "resolution": None, + # "slider_1": { + # "label": "Noise Augmentation", + # "value": 0.0, + # "minimum": 0.0, + # "maximum": 1.0 + # }, + # "slider_2": None, + # "slider_3": None, + # "priority": 0, + # "tags": [ + # "Revision" + # ] + # }, "scribble_hed": { "label": "scribble_hed", "call_function": scribble_hed, diff --git a/extensions-builtin/forge_preprocessor_clipvision/scripts/preprocessor_clipvision.py b/extensions-builtin/forge_preprocessor_clipvision/scripts/preprocessor_clipvision.py index 179fedec..5cabd66d 100644 --- a/extensions-builtin/forge_preprocessor_clipvision/scripts/preprocessor_clipvision.py +++ b/extensions-builtin/forge_preprocessor_clipvision/scripts/preprocessor_clipvision.py @@ -22,12 +22,12 @@ add_supported_preprocessor(PreprocessorClipVisionForIPAdapter( add_supported_preprocessor(PreprocessorClipVisionForIPAdapter( name='CLIP-ViT-bigG (IPAdapter)', - url='https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/model.safetensors', + url='https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/image_encoder/model.safetensors', filename='CLIP-ViT-bigG.safetensors' )) add_supported_preprocessor(PreprocessorClipVisionForIPAdapter( name='CLIP-ViT-L (IPAdapter)', url='https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/pytorch_model.bin', - filename='CLIP-ViT-bigG.safetensors' + filename='CLIP-ViT-L.safetensors' )) diff --git a/extensions-builtin/forge_preprocessor_revision/scripts/preprocessor_revision.py b/extensions-builtin/forge_preprocessor_revision/scripts/preprocessor_revision.py new file mode 100644 index 00000000..46274b6d --- /dev/null +++ b/extensions-builtin/forge_preprocessor_revision/scripts/preprocessor_revision.py @@ -0,0 +1,101 @@ +import torch +import copy + +from modules_forge.supported_preprocessor import PreprocessorClipVision, PreprocessorParameter +from modules_forge.shared import add_supported_preprocessor + + +def revision_conditioning_modifier(model, x, timestep, uncond, cond, cond_scale, model_options, seed): + revision_conditions = model_options['revision_conditions'] + noise_augmentor = model.noise_augmentor + noise_augment_merge = 0.0 + ignore_prompt = False + + adm_inputs = [] + weights = [] + noise_aug = [] + for revision_condition in revision_conditions: + adm_cond = revision_condition['cond'].image_embeds + weight = revision_condition["weight"] + noise_augment = revision_condition["noise_aug"] + noise_level = round((noise_augmentor.max_noise_level - 1) * noise_augment) + c_adm, noise_level_emb = noise_augmentor(adm_cond.to(x.device), + noise_level=torch.tensor([noise_level], device=x.device), seed=seed) + adm_out = torch.cat((c_adm, noise_level_emb), 1) * weight + weights.append(weight) + noise_aug.append(noise_augment) + adm_inputs.append(adm_out) + if revision_condition["ignore_prompt"]: + ignore_prompt = True + + if len(noise_aug) > 1: + adm_out = torch.stack(adm_inputs).sum(0) + noise_augment = noise_augment_merge + noise_level = round((noise_augmentor.max_noise_level - 1) * noise_augment) + c_adm, noise_level_emb = noise_augmentor(adm_out[:, :noise_augmentor.time_embed.dim], + noise_level=torch.tensor([noise_level], device=x.device)) + adm_out = torch.cat((c_adm, noise_level_emb), 1) + + cond = copy.deepcopy(cond) + uncond = copy.deepcopy(uncond) + + for c in cond: + a = 0 + + for c in uncond: + a = 0 + + if ignore_prompt: + for c in cond + uncond: + a = 0 + + return model, x, timestep, uncond, cond, cond_scale, model_options, seed + + +class PreprocessorClipVisionForRevision(PreprocessorClipVision): + def __init__(self, name, url, filename, ignore_prompt=False): + super().__init__(name, url, filename) + self.tags = ['Revision'] + self.model_filename_filters = ['Revision'] + self.do_not_need_model = True + self.ignore_prompt = ignore_prompt + self.slider_1 = PreprocessorParameter( + label="Noise Augmentation", minimum=0.0, maximum=1.0, value=0.0, visible=True) + + def process_before_every_sampling(self, process, cond, *args, **kwargs): + unit = kwargs['unit'] + + weight = float(unit.weight) + noise_aug = float(unit.threshold_a) + + unet = process.sd_model.forge_objects.unet.clone() + + if 'revision_conditions' not in unet.model_options: + unet.model_options['revision_conditions'] = [] + + unet.model_options['revision_conditions'].append(dict( + cond=cond, + weight=weight, + noise_aug=noise_aug, + ignore_prompt=self.ignore_prompt + )) + + unet.add_conditioning_modifier(revision_conditioning_modifier, ensure_uniqueness=True) + + process.sd_model.forge_objects.unet = unet + return + + +add_supported_preprocessor(PreprocessorClipVisionForRevision( + name='CLIP-G (Revision)', + url='https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/image_encoder/model.safetensors', + filename='CLIP-ViT-bigG.safetensors', + ignore_prompt=False +)) + +add_supported_preprocessor(PreprocessorClipVisionForRevision( + name='CLIP-G (Revision ignore prompt)', + url='https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/image_encoder/model.safetensors', + filename='CLIP-ViT-bigG.safetensors', + ignore_prompt=True +)) diff --git a/extensions-builtin/sd_forge_controlnet/scripts/controlnet.py b/extensions-builtin/sd_forge_controlnet/scripts/controlnet.py index 635a15d9..a8dfc566 100644 --- a/extensions-builtin/sd_forge_controlnet/scripts/controlnet.py +++ b/extensions-builtin/sd_forge_controlnet/scripts/controlnet.py @@ -23,6 +23,7 @@ import functools from PIL import Image from modules_forge.shared import try_load_supported_control_model +from modules_forge.supported_controlnet import ControlModelPatcher # Gradio 3.32 bug fix import tempfile @@ -455,9 +456,14 @@ class ControlNetForForgeOfficial(scripts.Script): params.control_cond_for_hr_fix = preprocessor_output p.extra_result_images.append(input_image) - model_filename = global_state.get_controlnet_filename(unit.model) + if preprocessor.do_not_need_model: + model_filename = 'Not Needed' + params.model = ControlModelPatcher() + else: + model_filename = global_state.get_controlnet_filename(unit.model) + params.model = cached_controlnet_loader(model_filename) + assert params.model is not None, logger.error(f"Recognizing Control Model failed: {model_filename}") - params.model = cached_controlnet_loader(model_filename) params.preprocessor = preprocessor params.preprocessor.process_after_running_preprocessors(process=p, params=params, **kwargs) diff --git a/modules_forge/forge_sampler.py b/modules_forge/forge_sampler.py index 010e24f3..d7db6531 100644 --- a/modules_forge/forge_sampler.py +++ b/modules_forge/forge_sampler.py @@ -76,6 +76,9 @@ def forge_sample(self, denoiser_params, cond_scale, cond_composition): for h in cond + uncond: h['control'] = control + for modifier in model_options.get('conditioning_modifiers', []): + model, x, timestep, uncond, cond, cond_scale, model_options, seed = modifier(model, x, timestep, uncond, cond, cond_scale, model_options, seed) + denoised = sampling_function(model, x, timestep, uncond, cond, cond_scale, model_options, seed) return denoised diff --git a/modules_forge/unet_patcher.py b/modules_forge/unet_patcher.py index 53df477e..8841cc6f 100644 --- a/modules_forge/unet_patcher.py +++ b/modules_forge/unet_patcher.py @@ -33,3 +33,13 @@ class UnetPatcher(ModelPatcher): results.append(pointer) pointer = pointer.previous_controlnet return results + + def add_conditioning_modifier(self, modifier, ensure_uniqueness=False): + if 'conditioning_modifiers' not in self.model_options: + self.model_options['conditioning_modifiers'] = [] + + if ensure_uniqueness and modifier in self.model_options['conditioning_modifiers']: + return + + self.model_options['conditioning_modifiers'].append(modifier) + return