diff --git a/backend/misc/checkpoint_pickle.py b/backend/misc/checkpoint_pickle.py new file mode 100644 index 00000000..664108ed --- /dev/null +++ b/backend/misc/checkpoint_pickle.py @@ -0,0 +1,15 @@ +import pickle + +load = pickle.load + + +class Empty: + pass + + +class Unpickler(pickle.Unpickler): + def find_class(self, module, name): + # TODO: safe unpickle + if module.startswith("pytorch_lightning"): + return Empty + return super().find_class(module, name) diff --git a/backend/patcher/clipvision.py b/backend/patcher/clipvision.py new file mode 100644 index 00000000..c9e4e3be --- /dev/null +++ b/backend/patcher/clipvision.py @@ -0,0 +1,190 @@ +import torch + +from backend.utils import load_torch_file +from backend.state_dict import transformers_convert, state_dict_prefix_replace +from backend import operations, memory_management +from backend.patcher.base import ModelPatcher +from transformers import modeling_utils, CLIPVisionConfig, CLIPVisionModelWithProjection + + +CLIP_VISION_G = { + "attention_dropout": 0.0, + "dropout": 0.0, + "hidden_act": "gelu", + "hidden_size": 1664, + "image_size": 224, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 8192, + "layer_norm_eps": 1e-05, + "model_type": "clip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 48, + "patch_size": 14, + "projection_dim": 1280, + "torch_dtype": "float32" +} + +CLIP_VISION_H = { + "attention_dropout": 0.0, + "dropout": 0.0, + "hidden_act": "gelu", + "hidden_size": 1280, + "image_size": 224, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 5120, + "layer_norm_eps": 1e-05, + "model_type": "clip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 32, + "patch_size": 14, + "projection_dim": 1024, + "torch_dtype": "float32" +} + + +CLIP_VISION_VITL = { + "attention_dropout": 0.0, + "dropout": 0.0, + "hidden_act": "quick_gelu", + "hidden_size": 1024, + "image_size": 224, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 4096, + "layer_norm_eps": 1e-05, + "model_type": "clip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 24, + "patch_size": 14, + "projection_dim": 768, + "torch_dtype": "float32" +} + + +class Output: + def __getitem__(self, key): + return getattr(self, key) + + def __setitem__(self, key, item): + setattr(self, key, item) + + +def clip_preprocess(image, size=224): + mean = torch.tensor([0.48145466, 0.4578275, 0.40821073], device=image.device, dtype=image.dtype) + std = torch.tensor([0.26862954, 0.26130258, 0.27577711], device=image.device, dtype=image.dtype) + image = image.movedim(-1, 1) + if not (image.shape[2] == size and image.shape[3] == size): + scale = (size / min(image.shape[2], image.shape[3])) + image = torch.nn.functional.interpolate(image, size=(round(scale * image.shape[2]), round(scale * image.shape[3])), mode="bicubic", antialias=True) + h = (image.shape[2] - size) // 2 + w = (image.shape[3] - size) // 2 + image = image[:, :, h:h + size, w:w + size] + image = torch.clip((255. * image), 0, 255).round() / 255.0 + return (image - mean.view([3, 1, 1])) / std.view([3, 1, 1]) + + +class ClipVisionModel: + def __init__(self, config): + config = CLIPVisionConfig(**config) + + self.load_device = memory_management.text_encoder_device() + self.offload_device = memory_management.text_encoder_offload_device() + + if memory_management.should_use_fp16(self.load_device, prioritize_performance=False): + self.dtype = torch.float16 + else: + self.dtype = torch.float32 + + with operations.using_forge_operations(): + with modeling_utils.no_init_weights(): + self.model = CLIPVisionModelWithProjection(config) + + self.model.to(self.dtype) + self.patcher = ModelPatcher( + self.model, + load_device=self.load_device, + offload_device=self.offload_device + ) + + def load_sd(self, sd): + return self.model.load_state_dict(sd, strict=False) + + def get_sd(self): + return self.model.state_dict() + + def encode_image(self, image): + memory_management.load_model_gpu(self.patcher) + pixel_values = clip_preprocess(image.to(self.load_device)) + outputs = self.model(pixel_values=pixel_values, output_hidden_states=True) + + o = Output() + o["last_hidden_state"] = outputs.last_hidden_state.to(memory_management.intermediate_device()) + o["penultimate_hidden_states"] = outputs.hidden_states[-2].to(memory_management.intermediate_device()) + o["image_embeds"] = outputs.image_embeds.to(memory_management.intermediate_device()) + + return o + + +def convert_to_transformers(sd, prefix): + sd_k = sd.keys() + if "{}transformer.resblocks.0.attn.in_proj_weight".format(prefix) in sd_k: + keys_to_replace = { + "{}class_embedding".format(prefix): "vision_model.embeddings.class_embedding", + "{}conv1.weight".format(prefix): "vision_model.embeddings.patch_embedding.weight", + "{}positional_embedding".format(prefix): "vision_model.embeddings.position_embedding.weight", + "{}ln_post.bias".format(prefix): "vision_model.post_layernorm.bias", + "{}ln_post.weight".format(prefix): "vision_model.post_layernorm.weight", + "{}ln_pre.bias".format(prefix): "vision_model.pre_layrnorm.bias", + "{}ln_pre.weight".format(prefix): "vision_model.pre_layrnorm.weight", + } + + for x in keys_to_replace: + if x in sd_k: + sd[keys_to_replace[x]] = sd.pop(x) + + if "{}proj".format(prefix) in sd_k: + sd['visual_projection.weight'] = sd.pop("{}proj".format(prefix)).transpose(0, 1) + + sd = transformers_convert(sd, prefix, "vision_model.", 48) + else: + replace_prefix = {prefix: ""} + sd = state_dict_prefix_replace(sd, replace_prefix) + return sd + + +def load_clipvision_from_sd(sd, prefix="", convert_keys=False): + if convert_keys: + sd = convert_to_transformers(sd, prefix) + if "vision_model.encoder.layers.47.layer_norm1.weight" in sd: + config = CLIP_VISION_G + elif "vision_model.encoder.layers.30.layer_norm1.weight" in sd: + config = CLIP_VISION_H + elif "vision_model.encoder.layers.22.layer_norm1.weight" in sd: + config = CLIP_VISION_VITL + else: + return None + + clip = ClipVisionModel(config) + m, u = clip.load_sd(sd) + if len(m) > 0: + print("extra clip vision:", m) + u = set(u) + keys = list(sd.keys()) + for k in keys: + if k not in u: + t = sd.pop(k) + del t + return clip + + +def load(ckpt_path): + sd = load_torch_file(ckpt_path) + if "visual.transformer.resblocks.0.attn.in_proj_weight" in sd: + return load_clipvision_from_sd(sd, prefix="visual.", convert_keys=True) + else: + return load_clipvision_from_sd(sd) diff --git a/backend/utils.py b/backend/utils.py index bd3d751d..f6c8ac1d 100644 --- a/backend/utils.py +++ b/backend/utils.py @@ -1,4 +1,29 @@ import torch +import safetensors.torch +import backend.misc.checkpoint_pickle + + +def load_torch_file(ckpt, safe_load=False, device=None): + if device is None: + device = torch.device("cpu") + if ckpt.lower().endswith(".safetensors"): + sd = safetensors.torch.load_file(ckpt, device=device.type) + else: + if safe_load: + if not 'weights_only' in torch.load.__code__.co_varnames: + print("Warning torch.load doesn't support weights_only on this pytorch version, loading unsafely.") + safe_load = False + if safe_load: + pl_sd = torch.load(ckpt, map_location=device, weights_only=True) + else: + pl_sd = torch.load(ckpt, map_location=device, pickle_module=backend.misc.checkpoint_pickle) + if "global_step" in pl_sd: + print(f"Global Step: {pl_sd['global_step']}") + if "state_dict" in pl_sd: + sd = pl_sd["state_dict"] + else: + sd = pl_sd + return sd def set_attr(obj, attr, value): diff --git a/extensions-builtin/forge_legacy_preprocessors/annotator/pidinet/__init__.py b/extensions-builtin/forge_legacy_preprocessors/annotator/pidinet/__init__.py index 64195e9b..8b913f93 100644 --- a/extensions-builtin/forge_legacy_preprocessors/annotator/pidinet/__init__.py +++ b/extensions-builtin/forge_legacy_preprocessors/annotator/pidinet/__init__.py @@ -6,7 +6,7 @@ from annotator.pidinet.model import pidinet from annotator.util import safe_step from modules import devices from annotator.annotator_path import models_path -from ldm_patched.modules.utils import load_torch_file +from backend.utils import load_torch_file netNetwork = None remote_model_path = "https://huggingface.co/lllyasviel/Annotators/resolve/main/table5_pidinet.pth" diff --git a/extensions-builtin/forge_preprocessor_reference/scripts/forge_reference.py b/extensions-builtin/forge_preprocessor_reference/scripts/forge_reference.py index cc06ffba..5a53171b 100644 --- a/extensions-builtin/forge_preprocessor_reference/scripts/forge_reference.py +++ b/extensions-builtin/forge_preprocessor_reference/scripts/forge_reference.py @@ -2,15 +2,15 @@ import torch from modules_forge.supported_preprocessor import Preprocessor, PreprocessorParameter from modules_forge.shared import add_supported_preprocessor -from ldm_patched.modules.samplers import sampling_function -import ldm_patched.ldm.modules.attention as attention +from backend.sampling.sampling_function import sampling_function_inner +from backend import attention def sdp(q, k, v, transformer_options): if q.shape[0] == 0: return q - return attention.optimized_attention(q, k, v, heads=transformer_options["n_heads"], mask=None) + return attention.attention_function(q, k, v, heads=transformer_options["n_heads"], mask=None) def adain(x, target_std, target_mean): @@ -81,7 +81,7 @@ class PreprocessorReference(Preprocessor): self.is_recording_style = True xt = latent_image.to(x) + torch.randn(x.size(), dtype=x.dtype, generator=gen_cpu).to(x) * sigma - sampling_function(model, xt, timestep, uncond, cond, 1, model_options, seed) + sampling_function_inner(model, xt, timestep, uncond, cond, 1, model_options, seed) self.is_recording_style = False diff --git a/extensions-builtin/sd_forge_controlllite/lib_controllllite/lib_controllllite.py b/extensions-builtin/sd_forge_controlllite/lib_controllllite/lib_controllllite.py index 73c45921..3ef4f72c 100644 --- a/extensions-builtin/sd_forge_controlllite/lib_controllllite/lib_controllllite.py +++ b/extensions-builtin/sd_forge_controlllite/lib_controllllite/lib_controllllite.py @@ -1,8 +1,5 @@ import math import torch -import os - -import ldm_patched.modules def extra_options_to_module_prefix(extra_options): diff --git a/extensions-builtin/sd_forge_fooocus_inpaint/scripts/forge_fooocus_inpaint.py b/extensions-builtin/sd_forge_fooocus_inpaint/scripts/forge_fooocus_inpaint.py index c62b2240..1ffe3513 100644 --- a/extensions-builtin/sd_forge_fooocus_inpaint/scripts/forge_fooocus_inpaint.py +++ b/extensions-builtin/sd_forge_fooocus_inpaint/scripts/forge_fooocus_inpaint.py @@ -1,14 +1,14 @@ import os import torch import copy +import backend.patcher.base from modules_forge.shared import add_supported_control_model from modules_forge.supported_controlnet import ControlModelPatcher from modules_forge.forge_sampler import sampling_prepare -from ldm_patched.modules.utils import load_torch_file -from ldm_patched.modules import model_patcher -from ldm_patched.modules.model_management import cast_to_device, current_loaded_models -from ldm_patched.modules.lora import model_lora_keys_unet +from backend.utils import load_torch_file +from backend.memory_management import cast_to_device, current_loaded_models +from backend.patcher.lora import model_lora_keys_unet def is_model_loaded(model): @@ -127,5 +127,5 @@ class FooocusInpaintPatcher(ControlModelPatcher): return -model_patcher.extra_weight_calculators['fooocus'] = calculate_weight_fooocus +backend.patcher.base.extra_weight_calculators['fooocus'] = calculate_weight_fooocus add_supported_control_model(FooocusInpaintPatcher) diff --git a/extensions-builtin/sd_forge_freeu/scripts/forge_freeu.py b/extensions-builtin/sd_forge_freeu/scripts/forge_freeu.py index 0a93ff3a..6240563e 100644 --- a/extensions-builtin/sd_forge_freeu/scripts/forge_freeu.py +++ b/extensions-builtin/sd_forge_freeu/scripts/forge_freeu.py @@ -1,47 +1,92 @@ +import torch import gradio as gr from modules import scripts -from ldm_patched.contrib.external_freelunch import FreeU_V2 + + +def Fourier_filter(x, threshold, scale): + # FFT + x_freq = torch.fft.fftn(x.float(), dim=(-2, -1)) + x_freq = torch.fft.fftshift(x_freq, dim=(-2, -1)) + + B, C, H, W = x_freq.shape + mask = torch.ones((B, C, H, W), device=x.device) + + crow, ccol = H // 2, W // 2 + mask[..., crow - threshold:crow + threshold, ccol - threshold:ccol + threshold] = scale + x_freq = x_freq * mask + + # IFFT + x_freq = torch.fft.ifftshift(x_freq, dim=(-2, -1)) + x_filtered = torch.fft.ifftn(x_freq, dim=(-2, -1)).real + + return x_filtered.to(x.dtype) + + +class FreeU: + def patch(self, model, b1, b2, s1, s2): + model_channels = model.model.model_config.unet_config["model_channels"] + scale_dict = {model_channels * 4: (b1, s1), model_channels * 2: (b2, s2)} + on_cpu_devices = {} + + def output_block_patch(h, hsp, transformer_options): + scale = scale_dict.get(h.shape[1], None) + if scale is not None: + h[:, :h.shape[1] // 2] = h[:, :h.shape[1] // 2] * scale[0] + if hsp.device not in on_cpu_devices: + try: + hsp = Fourier_filter(hsp, threshold=1, scale=scale[1]) + except: + print("Device", hsp.device, "does not support the torch.fft functions used in the FreeU node, switching to CPU.") + on_cpu_devices[hsp.device] = True + hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device) + else: + hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device) + + return h, hsp + + m = model.clone() + m.set_model_output_block_patch(output_block_patch) + return (m,) + + +class FreeU_V2: + def patch(self, model, b1, b2, s1, s2): + model_channels = model.model.diffusion_model.legacy_config["model_channels"] + scale_dict = {model_channels * 4: (b1, s1), model_channels * 2: (b2, s2)} + on_cpu_devices = {} + + def output_block_patch(h, hsp, transformer_options): + scale = scale_dict.get(h.shape[1], None) + if scale is not None: + hidden_mean = h.mean(1).unsqueeze(1) + B = hidden_mean.shape[0] + hidden_max, _ = torch.max(hidden_mean.view(B, -1), dim=-1, keepdim=True) + hidden_min, _ = torch.min(hidden_mean.view(B, -1), dim=-1, keepdim=True) + hidden_mean = (hidden_mean - hidden_min.unsqueeze(2).unsqueeze(3)) / (hidden_max - hidden_min).unsqueeze(2).unsqueeze(3) + + h[:, :h.shape[1] // 2] = h[:, :h.shape[1] // 2] * ((scale[0] - 1) * hidden_mean + 1) + + if hsp.device not in on_cpu_devices: + try: + hsp = Fourier_filter(hsp, threshold=1, scale=scale[1]) + except: + print("Device", hsp.device, "does not support the torch.fft functions used in the FreeU node, switching to CPU.") + on_cpu_devices[hsp.device] = True + hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device) + else: + hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device) + + return h, hsp + + m = model.clone() + m.set_model_output_block_patch(output_block_patch) + return (m,) opFreeU_V2 = FreeU_V2() -# def Fourier_filter(x, threshold, scale): -# x_freq = torch.fft.fftn(x.float(), dim=(-2, -1)) -# x_freq = torch.fft.fftshift(x_freq, dim=(-2, -1)) -# B, C, H, W = x_freq.shape -# mask = torch.ones((B, C, H, W), device=x.device) -# crow, ccol = H // 2, W //2 -# mask[..., crow - threshold:crow + threshold, ccol - threshold:ccol + threshold] = scale -# x_freq = x_freq * mask -# x_freq = torch.fft.ifftshift(x_freq, dim=(-2, -1)) -# x_filtered = torch.fft.ifftn(x_freq, dim=(-2, -1)).real -# return x_filtered.to(x.dtype) -# -# -# def set_freeu_v2_patch(model, b1, b2, s1, s2): -# model_channels = model.model.model_config.unet_config["model_channels"] -# scale_dict = {model_channels * 4: (b1, s1), model_channels * 2: (b2, s2)} -# -# def output_block_patch(h, hsp, *args, **kwargs): -# scale = scale_dict.get(h.shape[1], None) -# if scale is not None: -# hidden_mean = h.mean(1).unsqueeze(1) -# B = hidden_mean.shape[0] -# hidden_max, _ = torch.max(hidden_mean.view(B, -1), dim=-1, keepdim=True) -# hidden_min, _ = torch.min(hidden_mean.view(B, -1), dim=-1, keepdim=True) -# hidden_mean = (hidden_mean - hidden_min.unsqueeze(2).unsqueeze(3)) / \ -# (hidden_max - hidden_min).unsqueeze(2).unsqueeze(3) -# h[:, :h.shape[1] // 2] = h[:, :h.shape[1] // 2] * ((scale[0] - 1) * hidden_mean + 1) -# hsp = Fourier_filter(hsp, threshold=1, scale=scale[1]) -# return h, hsp -# -# m = model.clone() -# m.set_model_output_block_patch(output_block_patch) -# return m - - class FreeUForForge(scripts.Script): sorting_priority = 12 @@ -54,7 +99,7 @@ class FreeUForForge(scripts.Script): def ui(self, *args, **kwargs): with gr.Accordion(open=False, label=self.title(), elem_id="extensions-freeu", - elem_classes=["extensions-freeu"]): + elem_classes=["extensions-freeu"]): freeu_enabled = gr.Checkbox(label='Enabled', value=False) freeu_b1 = gr.Slider(label='B1', minimum=0, maximum=2, step=0.01, value=1.01) freeu_b2 = gr.Slider(label='B2', minimum=0, maximum=2, step=0.01, value=1.02) diff --git a/extensions-builtin/sd_forge_ipadapter/lib_ipadapter/IPAdapterPlus.py b/extensions-builtin/sd_forge_ipadapter/lib_ipadapter/IPAdapterPlus.py index dcced0b2..3abe9a16 100644 --- a/extensions-builtin/sd_forge_ipadapter/lib_ipadapter/IPAdapterPlus.py +++ b/extensions-builtin/sd_forge_ipadapter/lib_ipadapter/IPAdapterPlus.py @@ -1,15 +1,16 @@ -# https://github.com/cubiq/ComfyUI_IPAdapter_plus/blob/main/IPAdapterPlus.py +# https://github.com/cubiq/ComfyUI_IPAdapter_plus/blob/main/IPAdapterPlus.py from some early version +# Then maintained by Forge to add InstanceID and many other things + import torch import contextlib import os import math -import ldm_patched.modules.utils -import ldm_patched.modules.model_management -from ldm_patched.modules.clip_vision import clip_preprocess -from ldm_patched.ldm.modules.attention import optimized_attention -from ldm_patched.utils import path_utils as folder_paths +from backend import memory_management, attention, utils +from backend.misc.image_resize import adaptive_resize +from backend.patcher.clipvision import clip_preprocess +from modules_forge.shared import controlnet_dir, models_path from torch import nn from PIL import Image @@ -18,31 +19,25 @@ import torchvision.transforms as TT from lib_ipadapter.resampler import PerceiverAttention, FeedForward, Resampler -# set the models directory backward compatible -GLOBAL_MODELS_DIR = os.path.join(folder_paths.models_dir, "ipadapter") -MODELS_DIR = GLOBAL_MODELS_DIR if os.path.isdir(GLOBAL_MODELS_DIR) else os.path.join(os.path.dirname(os.path.realpath(__file__)), "models") -if "ipadapter" not in folder_paths.folder_names_and_paths: - current_paths = [MODELS_DIR] -else: - current_paths, _ = folder_paths.folder_names_and_paths["ipadapter"] -folder_paths.folder_names_and_paths["ipadapter"] = (current_paths, folder_paths.supported_pt_extensions) +GLOBAL_MODELS_DIR = os.path.join(models_path, "ipadapter") +MODELS_DIR = GLOBAL_MODELS_DIR +INSIGHTFACE_DIR = os.path.join(models_path, "insightface") -INSIGHTFACE_DIR = os.path.join(folder_paths.models_dir, "insightface") class FacePerceiverResampler(torch.nn.Module): def __init__( - self, - *, - dim=768, - depth=4, - dim_head=64, - heads=16, - embedding_dim=1280, - output_dim=768, - ff_mult=4, + self, + *, + dim=768, + depth=4, + dim_head=64, + heads=16, + embedding_dim=1280, + output_dim=768, + ff_mult=4, ): super().__init__() - + self.proj_in = torch.nn.Linear(embedding_dim, dim) self.proj_out = torch.nn.Linear(dim, output_dim) self.norm_out = torch.nn.LayerNorm(output_dim) @@ -65,21 +60,23 @@ class FacePerceiverResampler(torch.nn.Module): latents = self.proj_out(latents) return self.norm_out(latents) + class MLPProjModel(torch.nn.Module): def __init__(self, cross_attention_dim=1024, clip_embeddings_dim=1024): super().__init__() - + self.proj = torch.nn.Sequential( torch.nn.Linear(clip_embeddings_dim, clip_embeddings_dim), torch.nn.GELU(), torch.nn.Linear(clip_embeddings_dim, cross_attention_dim), torch.nn.LayerNorm(cross_attention_dim) ) - + def forward(self, image_embeds): clip_extra_context_tokens = self.proj(image_embeds) return clip_extra_context_tokens + class MLPProjModelFaceId(torch.nn.Module): def __init__(self, cross_attention_dim=768, id_embeddings_dim=512, num_tokens=4): super().__init__() @@ -88,9 +85,9 @@ class MLPProjModelFaceId(torch.nn.Module): self.num_tokens = num_tokens self.proj = torch.nn.Sequential( - torch.nn.Linear(id_embeddings_dim, id_embeddings_dim*2), + torch.nn.Linear(id_embeddings_dim, id_embeddings_dim * 2), torch.nn.GELU(), - torch.nn.Linear(id_embeddings_dim*2, cross_attention_dim*num_tokens), + torch.nn.Linear(id_embeddings_dim * 2, cross_attention_dim * num_tokens), ) self.norm = torch.nn.LayerNorm(cross_attention_dim) @@ -100,20 +97,21 @@ class MLPProjModelFaceId(torch.nn.Module): clip_extra_context_tokens = self.norm(clip_extra_context_tokens) return clip_extra_context_tokens + class ProjModelFaceIdPlus(torch.nn.Module): def __init__(self, cross_attention_dim=768, id_embeddings_dim=512, clip_embeddings_dim=1280, num_tokens=4): super().__init__() self.cross_attention_dim = cross_attention_dim self.num_tokens = num_tokens - + self.proj = torch.nn.Sequential( - torch.nn.Linear(id_embeddings_dim, id_embeddings_dim*2), + torch.nn.Linear(id_embeddings_dim, id_embeddings_dim * 2), torch.nn.GELU(), - torch.nn.Linear(id_embeddings_dim*2, cross_attention_dim*num_tokens), + torch.nn.Linear(id_embeddings_dim * 2, cross_attention_dim * num_tokens), ) self.norm = torch.nn.LayerNorm(cross_attention_dim) - + self.perceiver_resampler = FacePerceiverResampler( dim=cross_attention_dim, depth=4, @@ -123,7 +121,7 @@ class ProjModelFaceIdPlus(torch.nn.Module): output_dim=cross_attention_dim, ff_mult=4, ) - + def forward(self, id_embeds, clip_embeds, scale=1.0, shortcut=False): x = self.proj(id_embeds) x = x.reshape(-1, self.num_tokens, self.cross_attention_dim) @@ -133,21 +131,23 @@ class ProjModelFaceIdPlus(torch.nn.Module): out = x + scale * out return out + class ImageProjModel(nn.Module): def __init__(self, cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4): super().__init__() - + self.cross_attention_dim = cross_attention_dim self.clip_extra_context_tokens = clip_extra_context_tokens self.proj = nn.Linear(clip_embeddings_dim, self.clip_extra_context_tokens * cross_attention_dim) self.norm = nn.LayerNorm(cross_attention_dim) - + def forward(self, image_embeds): embeds = image_embeds clip_extra_context_tokens = self.proj(embeds).reshape(-1, self.clip_extra_context_tokens, self.cross_attention_dim) clip_extra_context_tokens = self.norm(clip_extra_context_tokens) return clip_extra_context_tokens + class To_KV(nn.Module): def __init__(self, state_dict): super().__init__() @@ -157,6 +157,7 @@ class To_KV(nn.Module): self.to_kvs[key.replace(".weight", "").replace(".", "_")] = nn.Linear(value.shape[1], value.shape[0], bias=False) self.to_kvs[key.replace(".weight", "").replace(".", "_")].weight.data = value + def set_model_patch_replace(model, patch_kwargs, key): to = model.model_options["transformer_options"] if "patches_replace" not in to: @@ -169,41 +170,46 @@ def set_model_patch_replace(model, patch_kwargs, key): else: to["patches_replace"]["attn2"][key].set_new_condition(**patch_kwargs) + def image_add_noise(image, noise): - image = image.permute([0,3,1,2]) - torch.manual_seed(0) # use a fixed random for reproducible results + image = image.permute([0, 3, 1, 2]) + torch.manual_seed(0) # use a fixed random for reproducible results transforms = TT.Compose([ TT.CenterCrop(min(image.shape[2], image.shape[3])), TT.Resize((224, 224), interpolation=TT.InterpolationMode.BICUBIC, antialias=True), - TT.ElasticTransform(alpha=75.0, sigma=noise*3.5), # shuffle the image - TT.RandomVerticalFlip(p=1.0), # flip the image to change the geometry even more + TT.ElasticTransform(alpha=75.0, sigma=noise * 3.5), # shuffle the image + TT.RandomVerticalFlip(p=1.0), # flip the image to change the geometry even more TT.RandomHorizontalFlip(p=1.0), ]) image = transforms(image.cpu()) - image = image.permute([0,2,3,1]) - image = image + ((0.25*(1-noise)+0.05) * torch.randn_like(image) ) # add further random noise + image = image.permute([0, 2, 3, 1]) + image = image + ((0.25 * (1 - noise) + 0.05) * torch.randn_like(image)) # add further random noise return image + def zeroed_hidden_states(clip_vision, batch_size): image = torch.zeros([batch_size, 224, 224, 3]) - ldm_patched.modules.model_management.load_model_gpu(clip_vision.patcher) + memory_management.load_model_gpu(clip_vision.patcher) pixel_values = clip_preprocess(image.to(clip_vision.load_device)).float() outputs = clip_vision.model(pixel_values=pixel_values, output_hidden_states=True) - outputs = outputs.hidden_states[-2].to(ldm_patched.modules.model_management.intermediate_device()) + outputs = outputs.hidden_states[-2].to(memory_management.intermediate_device()) return outputs + def min_(tensor_list): # return the element-wise min of the tensor list. x = torch.stack(tensor_list) mn = x.min(axis=0)[0] return torch.clamp(mn, min=0) - + + def max_(tensor_list): # return the element-wise max of the tensor list. x = torch.stack(tensor_list) mx = x.max(axis=0)[0] return torch.clamp(mx, max=1) + # From https://github.com/Jamy-L/Pytorch-Contrast-Adaptive-Sharpening/ def contrast_adaptive_sharpening(image, amount): img = F.pad(image, pad=(1, 1, 1, 1)).cpu() @@ -217,33 +223,34 @@ def contrast_adaptive_sharpening(image, amount): g = img[..., 2:, :-2] h = img[..., 2:, 1:-1] i = img[..., 2:, 2:] - + # Computing contrast cross = (b, d, e, f, h) mn = min_(cross) mx = max_(cross) - + diag = (a, c, g, i) mn2 = min_(diag) mx2 = max_(diag) mx = mx + mx2 mn = mn + mn2 - + # Computing local weight inv_mx = torch.reciprocal(mx) amp = inv_mx * torch.minimum(mn, (2 - mx)) # scaling amp = torch.sqrt(amp) - w = - amp * (amount * (1/5 - 1/8) + 1/8) - div = torch.reciprocal(1 + 4*w) + w = - amp * (amount * (1 / 5 - 1 / 8) + 1 / 8) + div = torch.reciprocal(1 + 4 * w) - output = ((b + d + f + h)*w + e) * div + output = ((b + d + f + h) * w + e) * div output = output.clamp(0, 1) output = torch.nan_to_num(output) return (output) + def tensorToNP(image): out = torch.clamp(255. * image.detach().cpu(), 0, 255).to(torch.uint8) out = out[..., [2, 1, 0]] @@ -251,13 +258,15 @@ def tensorToNP(image): return out + def NPToTensor(image): out = torch.from_numpy(image) - out = torch.clamp(out.to(torch.float)/255., 0.0, 1.0) + out = torch.clamp(out.to(torch.float) / 255., 0.0, 1.0) out = out[..., [2, 1, 0]] return out + class IPAdapter(nn.Module): def __init__(self, ipadapter_model, cross_attention_dim=1024, output_cross_attention_dim=1024, clip_embeddings_dim=1024, clip_extra_context_tokens=4, @@ -356,6 +365,7 @@ class IPAdapter(nn.Module): uc = self.image_proj_model(torch.zeros_like(prompt_image_emb)) return c, uc + class CrossAttentionPatch: # forward for patching def __init__(self, weight, ipadapter, number, cond, uncond, weight_type, mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False): @@ -370,9 +380,9 @@ class CrossAttentionPatch: self.sigma_end = [sigma_end] self.unfold_batch = [unfold_batch] - self.k_key = str(self.number*2+1) + "_to_k_ip" - self.v_key = str(self.number*2+1) + "_to_v_ip" - + self.k_key = str(self.number * 2 + 1) + "_to_k_ip" + self.v_key = str(self.number * 2 + 1) + "_to_v_ip" + def set_new_condition(self, weight, ipadapter, number, cond, uncond, weight_type, mask=None, sigma_start=0.0, sigma_end=1.0, unfold_batch=False): self.weights.append(weight) self.ipadapters.append(ipadapter) @@ -400,9 +410,9 @@ class CrossAttentionPatch: b = q.shape[0] qs = q.shape[1] batch_prompt = b // len(cond_or_uncond) - out = optimized_attention(q, k, v, extra_options["n_heads"]) + out = attention.attention_function(q, k, v, extra_options["n_heads"]) _, _, lh, lw = extra_options["original_shape"] - + for weight, cond, uncond, ipadapter, mask, weight_type, sigma_start, sigma_end, unfold_batch in zip(self.weights, self.conds, self.unconds, self.ipadapters, self.masks, self.weight_type, self.sigma_start, self.sigma_end, self.unfold_batch): if sigma > sigma_start or sigma < sigma_end: continue @@ -418,8 +428,8 @@ class CrossAttentionPatch: else: # check if images length matches full_length - if not, make it match if cond.shape[0] < ad_params["full_length"]: - cond = torch.cat((cond, cond[-1:].repeat((ad_params["full_length"]-cond.shape[0], 1, 1))), dim=0) - uncond = torch.cat((uncond, uncond[-1:].repeat((ad_params["full_length"]-uncond.shape[0], 1, 1))), dim=0) + cond = torch.cat((cond, cond[-1:].repeat((ad_params["full_length"] - cond.shape[0], 1, 1))), dim=0) + uncond = torch.cat((uncond, uncond[-1:].repeat((ad_params["full_length"] - uncond.shape[0], 1, 1))), dim=0) # if we have too many remove the excess (should not happen, but just in case) if cond.shape[0] > ad_params["full_length"]: cond = cond[:ad_params["full_length"]] @@ -429,8 +439,8 @@ class CrossAttentionPatch: # if we don't have enough reference images repeat the last one until we reach the right size if cond.shape[0] < batch_prompt: - cond = torch.cat((cond, cond[-1:].repeat((batch_prompt-cond.shape[0], 1, 1))), dim=0) - uncond = torch.cat((uncond, uncond[-1:].repeat((batch_prompt-uncond.shape[0], 1, 1))), dim=0) + cond = torch.cat((cond, cond[-1:].repeat((batch_prompt - cond.shape[0], 1, 1))), dim=0) + uncond = torch.cat((uncond, uncond[-1:].repeat((batch_prompt - uncond.shape[0], 1, 1))), dim=0) # if we have too many remove the exceeding elif cond.shape[0] > batch_prompt: cond = cond[:batch_prompt] @@ -464,7 +474,7 @@ class CrossAttentionPatch: ip_k = ip_k * W ip_v = ip_v_offset + ip_v_mean * W - out_ip = optimized_attention(q, ip_k.to(org_dtype), ip_v.to(org_dtype), extra_options["n_heads"]) + out_ip = attention.attention_function(q, ip_k.to(org_dtype), ip_v.to(org_dtype), extra_options["n_heads"]) if weight_type.startswith("original"): out_ip = out_ip * weight @@ -486,7 +496,7 @@ class CrossAttentionPatch: mask_downsample = F.interpolate(mask.unsqueeze(1), size=(mask_h, mask_w), mode="bicubic").squeeze(1) # check if mask length matches full_length - if not, make it match if mask_downsample.shape[0] < ad_params["full_length"]: - mask_downsample = torch.cat((mask_downsample, mask_downsample[-1:].repeat((ad_params["full_length"]-mask_downsample.shape[0], 1, 1))), dim=0) + mask_downsample = torch.cat((mask_downsample, mask_downsample[-1:].repeat((ad_params["full_length"] - mask_downsample.shape[0], 1, 1))), dim=0) # if we have too many remove the excess (should not happen, but just in case) if mask_downsample.shape[0] > ad_params["full_length"]: mask_downsample = mask_downsample[:ad_params["full_length"]] @@ -498,11 +508,11 @@ class CrossAttentionPatch: # if we don't have enough masks repeat the last one until we reach the right size if mask_downsample.shape[0] < batch_prompt: - mask_downsample = torch.cat((mask_downsample, mask_downsample[-1:, :, :].repeat((batch_prompt-mask_downsample.shape[0], 1, 1))), dim=0) + mask_downsample = torch.cat((mask_downsample, mask_downsample[-1:, :, :].repeat((batch_prompt - mask_downsample.shape[0], 1, 1))), dim=0) # if we have too many remove the exceeding elif mask_downsample.shape[0] > batch_prompt: mask_downsample = mask_downsample[:batch_prompt, :, :] - + # repeat the masks mask_downsample = mask_downsample.repeat(len(cond_or_uncond), 1, 1) mask_downsample = mask_downsample.view(mask_downsample.shape[0], -1, 1).repeat(1, 1, out.shape[2]) @@ -513,19 +523,12 @@ class CrossAttentionPatch: return out.to(dtype=org_dtype) + class IPAdapterModelLoader: - @classmethod - def INPUT_TYPES(s): - return {"required": { "ipadapter_file": (folder_paths.get_filename_list("ipadapter"), )}} - - RETURN_TYPES = ("IPADAPTER",) - FUNCTION = "load_ipadapter_model" - CATEGORY = "ipadapter" - def load_ipadapter_model(self, ipadapter_file): - ckpt_path = folder_paths.get_full_path("ipadapter", ipadapter_file) + ckpt_path = os.path.join(controlnet_dir, "ipadapter", ipadapter_file) - model = ldm_patched.modules.utils.load_torch_file(ckpt_path, safe_load=True) + model = utils.load_torch_file(ckpt_path, safe_load=True) if ckpt_path.lower().endswith(".safetensors"): st_model = {"image_proj": {}, "ip_adapter": {}} @@ -535,19 +538,22 @@ class IPAdapterModelLoader: elif key.startswith("ip_adapter."): st_model["ip_adapter"][key.replace("ip_adapter.", "")] = model[key] model = st_model - + if not "ip_adapter" in model.keys() or not model["ip_adapter"]: raise Exception("invalid IPAdapter model {}".format(ckpt_path)) return (model,) + insightface_face_align = None + + class InsightFaceLoader: @classmethod def INPUT_TYPES(s): return { "required": { - "provider": (["CPU", "CUDA", "ROCM"], ), + "provider": (["CPU", "CUDA", "ROCM"],), }, } @@ -577,47 +583,24 @@ class InsightFaceLoader: local_path = os.path.join(model_root, local_file) if not os.path.exists(local_path): load_file_from_url(url, model_dir=model_root) - + from insightface.utils import face_align global insightface_face_align insightface_face_align = face_align - model = FaceAnalysis(name=name, root=INSIGHTFACE_DIR, providers=[provider + 'ExecutionProvider',]) + model = FaceAnalysis(name=name, root=INSIGHTFACE_DIR, providers=[provider + 'ExecutionProvider', ]) model.prepare(ctx_id=0, det_size=(640, 640)) return (model,) + class IPAdapterApply: - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "ipadapter": ("IPADAPTER", ), - "clip_vision": ("CLIP_VISION",), - "image": ("IMAGE",), - "model": ("MODEL", ), - "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }), - "noise": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01 }), - "weight_type": (["original", "linear", "channel penalty"], ), - "start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "unfold_batch": ("BOOLEAN", { "default": False }), - }, - "optional": { - "attn_mask": ("MASK",), - } - } - - RETURN_TYPES = ("MODEL", ) - FUNCTION = "apply_ipadapter" - CATEGORY = "ipadapter" - def apply_ipadapter(self, ipadapter, model, weight, clip_vision=None, image=None, weight_type="original", noise=None, embeds=None, attn_mask=None, start_at=0.0, end_at=1.0, unfold_batch=False, insightface=None, faceid_v2=False, weight_v2=False, instant_id=False): - self.dtype = torch.float16 if ldm_patched.modules.model_management.should_use_fp16() else torch.float32 - self.device = ldm_patched.modules.model_management.get_torch_device() + self.dtype = torch.float16 if memory_management.should_use_fp16() else torch.float32 + self.device = memory_management.get_torch_device() self.weight = weight self.is_full = "proj.3.weight" in ipadapter["image_proj"] self.is_portrait = "proj.2.weight" in ipadapter["image_proj"] and not "proj.3.weight" in ipadapter["image_proj"] and not "0.to_q_lora.down.weight" in ipadapter["ip_adapter"] @@ -662,14 +645,14 @@ class IPAdapterApply: face_embed = torch.stack(face_embed, dim=0) clip_embed = face_embed elif self.is_faceid: - insightface.det_model.input_size = (640,640) # reset the detection size + insightface.det_model.input_size = (640, 640) # reset the detection size face_img = tensorToNP(image) face_embed = [] face_clipvision = [] for i in range(face_img.shape[0]): for size in [(size, size) for size in range(640, 128, -64)]: - insightface.det_model.input_size = size # TODO: hacky but seems to be working + insightface.det_model.input_size = size # TODO: hacky but seems to be working face = insightface.get(face_img[i]) if face: face_embed.append(torch.from_numpy(face[0].normed_embedding).unsqueeze(0)) @@ -692,7 +675,7 @@ class IPAdapterApply: clip_embed_zeroed = clip_vision.encode_image(neg_image).penultimate_hidden_states else: clip_embed_zeroed = zeroed_hidden_states(clip_vision, image.shape[0]) - + # TODO: check noise to the uncods too face_embed_zeroed = torch.zeros_like(face_embed) else: @@ -704,7 +687,7 @@ class IPAdapterApply: clip_embed = clip_vision.encode_image(image) neg_image = image_add_noise(image, noise) if noise > 0 else None - + if self.is_plus: clip_embed = clip_embed.penultimate_hidden_states if noise > 0: @@ -732,7 +715,7 @@ class IPAdapterApply: is_faceid=self.is_faceid, is_instant_id=self.is_instant_id ) - + self.ipadapter.to(self.device, dtype=self.dtype) if self.is_instant_id: @@ -777,21 +760,21 @@ class IPAdapterApply: } if not self.is_sdxl: - for id in [1,2,4,5,7,8]: # id of input_blocks that have cross attention + for id in [1, 2, 4, 5, 7, 8]: # id of input_blocks that have cross attention set_model_patch_replace(work_model, patch_kwargs, ("input", id)) patch_kwargs["number"] += 1 - for id in [3,4,5,6,7,8,9,10,11]: # id of output_blocks that have cross attention + for id in [3, 4, 5, 6, 7, 8, 9, 10, 11]: # id of output_blocks that have cross attention set_model_patch_replace(work_model, patch_kwargs, ("output", id)) patch_kwargs["number"] += 1 set_model_patch_replace(work_model, patch_kwargs, ("middle", 0)) else: - for id in [4,5,7,8]: # id of input_blocks that have cross attention - block_indices = range(2) if id in [4, 5] else range(10) # transformer_depth + for id in [4, 5, 7, 8]: # id of input_blocks that have cross attention + block_indices = range(2) if id in [4, 5] else range(10) # transformer_depth for index in block_indices: set_model_patch_replace(work_model, patch_kwargs, ("input", id, index)) patch_kwargs["number"] += 1 - for id in range(6): # id of output_blocks that have cross attention - block_indices = range(2) if id in [3, 4, 5] else range(10) # transformer_depth + for id in range(6): # id of output_blocks that have cross attention + block_indices = range(2) if id in [3, 4, 5] else range(10) # transformer_depth for index in block_indices: set_model_patch_replace(work_model, patch_kwargs, ("output", id, index)) patch_kwargs["number"] += 1 @@ -799,35 +782,37 @@ class IPAdapterApply: set_model_patch_replace(work_model, patch_kwargs, ("middle", 0, index)) patch_kwargs["number"] += 1 - return (work_model, ) + return (work_model,) + class IPAdapterApplyFaceID(IPAdapterApply): @classmethod def INPUT_TYPES(s): return { "required": { - "ipadapter": ("IPADAPTER", ), + "ipadapter": ("IPADAPTER",), "clip_vision": ("CLIP_VISION",), "insightface": ("INSIGHTFACE",), "image": ("IMAGE",), - "model": ("MODEL", ), - "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }), - "noise": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01 }), - "weight_type": (["original", "linear", "channel penalty"], ), - "start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "faceid_v2": ("BOOLEAN", { "default": False }), - "weight_v2": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }), - "unfold_batch": ("BOOLEAN", { "default": False }), + "model": ("MODEL",), + "weight": ("FLOAT", {"default": 1.0, "min": -1, "max": 3, "step": 0.05}), + "noise": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "weight_type": (["original", "linear", "channel penalty"],), + "start_at": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), + "end_at": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}), + "faceid_v2": ("BOOLEAN", {"default": False}), + "weight_v2": ("FLOAT", {"default": 1.0, "min": -1, "max": 3, "step": 0.05}), + "unfold_batch": ("BOOLEAN", {"default": False}), }, "optional": { "attn_mask": ("MASK",), } } -def prepImage(image, interpolation="LANCZOS", crop_position="center", size=(224,224), sharpening=0.0, padding=0): + +def prepImage(image, interpolation="LANCZOS", crop_position="center", size=(224, 224), sharpening=0.0, padding=0): _, oh, ow, _ = image.shape - output = image.permute([0,3,1,2]) + output = image.permute([0, 3, 1, 2]) if "pad" in crop_position: target_length = max(oh, ow) @@ -838,19 +823,19 @@ def prepImage(image, interpolation="LANCZOS", crop_position="center", size=(224, output = F.pad(output, (pad_l, pad_r, pad_t, pad_b), value=0, mode="constant") else: crop_size = min(oh, ow) - x = (ow-crop_size) // 2 - y = (oh-crop_size) // 2 + x = (ow - crop_size) // 2 + y = (oh - crop_size) // 2 if "top" in crop_position: y = 0 elif "bottom" in crop_position: - y = oh-crop_size + y = oh - crop_size elif "left" in crop_position: x = 0 elif "right" in crop_position: - x = ow-crop_size - - x2 = x+crop_size - y2 = y+crop_size + x = ow - crop_size + + x2 = x + crop_size + y2 = y + crop_size # crop output = output[:, :, y:y2, x:x2] @@ -862,18 +847,19 @@ def prepImage(image, interpolation="LANCZOS", crop_position="center", size=(224, img = img.resize(size, resample=Image.Resampling[interpolation]) imgs.append(TT.ToTensor()(img)) output = torch.stack(imgs, dim=0) - imgs = None # zelous GC - + imgs = None # zelous GC + if sharpening > 0: output = contrast_adaptive_sharpening(output, sharpening) - + if padding > 0: output = F.pad(output, (padding, padding, padding, padding), value=255, mode="constant") - output = output.permute([0,2,3,1]) + output = output.permute([0, 2, 3, 1]) return output + class PrepImageForInsightFace: @classmethod def INPUT_TYPES(s): @@ -881,8 +867,8 @@ class PrepImageForInsightFace: "image": ("IMAGE",), "crop_position": (["center", "top", "bottom", "left", "right"],), "sharpening": ("FLOAT", {"default": 0.0, "min": 0, "max": 1, "step": 0.05}), - "pad_around": ("BOOLEAN", { "default": True }), - }, + "pad_around": ("BOOLEAN", {"default": True}), + }, } RETURN_TYPES = ("IMAGE",) @@ -899,7 +885,8 @@ class PrepImageForInsightFace: size = (640, 640) output = prepImage(image, "LANCZOS", crop_position, size, sharpening, padding) - return (output, ) + return (output,) + class PrepImageForClipVision: @classmethod @@ -909,7 +896,7 @@ class PrepImageForClipVision: "interpolation": (["LANCZOS", "BICUBIC", "HAMMING", "BILINEAR", "BOX", "NEAREST"],), "crop_position": (["top", "bottom", "left", "right", "center", "pad"],), "sharpening": ("FLOAT", {"default": 0.0, "min": 0, "max": 1, "step": 0.05}), - }, + }, } RETURN_TYPES = ("IMAGE",) @@ -920,7 +907,8 @@ class PrepImageForClipVision: def prep_image(self, image, interpolation="LANCZOS", crop_position="center", sharpening=0.0): size = (224, 224) output = prepImage(image, interpolation, crop_position, size, sharpening, 0) - return (output, ) + return (output,) + class IPAdapterEncoder: @classmethod @@ -928,17 +916,17 @@ class IPAdapterEncoder: return {"required": { "clip_vision": ("CLIP_VISION",), "image_1": ("IMAGE",), - "ipadapter_plus": ("BOOLEAN", { "default": False }), - "noise": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01 }), - "weight_1": ("FLOAT", { "default": 1.0, "min": 0, "max": 1.0, "step": 0.01 }), - }, + "ipadapter_plus": ("BOOLEAN", {"default": False}), + "noise": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "weight_1": ("FLOAT", {"default": 1.0, "min": 0, "max": 1.0, "step": 0.01}), + }, "optional": { "image_2": ("IMAGE",), "image_3": ("IMAGE",), "image_4": ("IMAGE",), - "weight_2": ("FLOAT", { "default": 1.0, "min": 0, "max": 1.0, "step": 0.01 }), - "weight_3": ("FLOAT", { "default": 1.0, "min": 0, "max": 1.0, "step": 0.01 }), - "weight_4": ("FLOAT", { "default": 1.0, "min": 0, "max": 1.0, "step": 0.01 }), + "weight_2": ("FLOAT", {"default": 1.0, "min": 0, "max": 1.0, "step": 0.01}), + "weight_3": ("FLOAT", {"default": 1.0, "min": 0, "max": 1.0, "step": 0.01}), + "weight_4": ("FLOAT", {"default": 1.0, "min": 0, "max": 1.0, "step": 0.01}), } } @@ -953,27 +941,27 @@ class IPAdapterEncoder: weight_4 *= (0.1 + (weight_4 - 0.1)) image = image_1 - weight = [weight_1]*image_1.shape[0] - + weight = [weight_1] * image_1.shape[0] + if image_2 is not None: if image_1.shape[1:] != image_2.shape[1:]: - image_2 = ldm_patched.modules.utils.common_upscale(image_2.movedim(-1,1), image.shape[2], image.shape[1], "bilinear", "center").movedim(1,-1) + image_2 = adaptive_resize(image_2.movedim(-1, 1), image.shape[2], image.shape[1], "bilinear", "center").movedim(1, -1) image = torch.cat((image, image_2), dim=0) - weight += [weight_2]*image_2.shape[0] + weight += [weight_2] * image_2.shape[0] if image_3 is not None: if image.shape[1:] != image_3.shape[1:]: - image_3 = ldm_patched.modules.utils.common_upscale(image_3.movedim(-1,1), image.shape[2], image.shape[1], "bilinear", "center").movedim(1,-1) + image_3 = adaptive_resize(image_3.movedim(-1, 1), image.shape[2], image.shape[1], "bilinear", "center").movedim(1, -1) image = torch.cat((image, image_3), dim=0) - weight += [weight_3]*image_3.shape[0] + weight += [weight_3] * image_3.shape[0] if image_4 is not None: if image.shape[1:] != image_4.shape[1:]: - image_4 = ldm_patched.modules.utils.common_upscale(image_4.movedim(-1,1), image.shape[2], image.shape[1], "bilinear", "center").movedim(1,-1) + image_4 = adaptive_resize(image_4.movedim(-1, 1), image.shape[2], image.shape[1], "bilinear", "center").movedim(1, -1) image = torch.cat((image, image_4), dim=0) - weight += [weight_4]*image_4.shape[0] - + weight += [weight_4] * image_4.shape[0] + clip_embed = clip_vision.encode_image(image) neg_image = image_add_noise(image, noise) if noise > 0 else None - + if ipadapter_plus: clip_embed = clip_embed.penultimate_hidden_states if noise > 0: @@ -990,113 +978,7 @@ class IPAdapterEncoder: if any(e != 1.0 for e in weight): weight = torch.tensor(weight).unsqueeze(-1) if not ipadapter_plus else torch.tensor(weight).unsqueeze(-1).unsqueeze(-1) clip_embed = clip_embed * weight - + output = torch.stack((clip_embed, clip_embed_zeroed)) - return( output, ) - -class IPAdapterApplyEncoded(IPAdapterApply): - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "ipadapter": ("IPADAPTER", ), - "embeds": ("EMBEDS",), - "model": ("MODEL", ), - "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }), - "weight_type": (["original", "linear", "channel penalty"], ), - "start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }), - "unfold_batch": ("BOOLEAN", { "default": False }), - }, - "optional": { - "attn_mask": ("MASK",), - } - } - -class IPAdapterSaveEmbeds: - def __init__(self): - self.output_dir = folder_paths.get_output_directory() - - @classmethod - def INPUT_TYPES(s): - return {"required": { - "embeds": ("EMBEDS",), - "filename_prefix": ("STRING", {"default": "embeds/IPAdapter"}) - }, - } - - RETURN_TYPES = () - FUNCTION = "save" - OUTPUT_NODE = True - CATEGORY = "ipadapter" - - def save(self, embeds, filename_prefix): - full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir) - file = f"{filename}_{counter:05}_.ipadpt" - file = os.path.join(full_output_folder, file) - - torch.save(embeds, file) - return (None, ) - - -class IPAdapterLoadEmbeds: - @classmethod - def INPUT_TYPES(s): - input_dir = folder_paths.get_input_directory() - files = [os.path.relpath(os.path.join(root, file), input_dir) for root, dirs, files in os.walk(input_dir) for file in files if file.endswith('.ipadpt')] - return {"required": {"embeds": [sorted(files), ]}, } - - RETURN_TYPES = ("EMBEDS", ) - FUNCTION = "load" - CATEGORY = "ipadapter" - - def load(self, embeds): - path = folder_paths.get_annotated_filepath(embeds) - output = torch.load(path).cpu() - - return (output, ) - - -class IPAdapterBatchEmbeds: - @classmethod - def INPUT_TYPES(s): - return {"required": { - "embed1": ("EMBEDS",), - "embed2": ("EMBEDS",), - }} - - RETURN_TYPES = ("EMBEDS",) - FUNCTION = "batch" - CATEGORY = "ipadapter" - - def batch(self, embed1, embed2): - return (torch.cat((embed1, embed2), dim=1), ) - -NODE_CLASS_MAPPINGS = { - "IPAdapterModelLoader": IPAdapterModelLoader, - "IPAdapterApply": IPAdapterApply, - "IPAdapterApplyFaceID": IPAdapterApplyFaceID, - "IPAdapterApplyEncoded": IPAdapterApplyEncoded, - "PrepImageForClipVision": PrepImageForClipVision, - "IPAdapterEncoder": IPAdapterEncoder, - "IPAdapterSaveEmbeds": IPAdapterSaveEmbeds, - "IPAdapterLoadEmbeds": IPAdapterLoadEmbeds, - "IPAdapterBatchEmbeds": IPAdapterBatchEmbeds, - "InsightFaceLoader": InsightFaceLoader, - "PrepImageForInsightFace": PrepImageForInsightFace, -} - -NODE_DISPLAY_NAME_MAPPINGS = { - "IPAdapterModelLoader": "Load IPAdapter Model", - "IPAdapterApply": "Apply IPAdapter", - "IPAdapterApplyFaceID": "Apply IPAdapter FaceID", - "IPAdapterApplyEncoded": "Apply IPAdapter from Encoded", - "PrepImageForClipVision": "Prepare Image For Clip Vision", - "IPAdapterEncoder": "Encode IPAdapter Image", - "IPAdapterSaveEmbeds": "Save IPAdapter Embeds", - "IPAdapterLoadEmbeds": "Load IPAdapter Embeds", - "IPAdapterBatchEmbeds": "IPAdapter Batch Embeds", - "InsightFaceLoader": "Load InsightFace", - "PrepImageForInsightFace": "Prepare Image For InsightFace", -} \ No newline at end of file + return (output,)