initial commit

extensions-builtin/sd_forge_ipadapter/scripts/forge_ipadapter.py

@@ -0,0 +1,163 @@
+from modules_forge.supported_preprocessor import PreprocessorClipVision, Preprocessor, PreprocessorParameter
+from modules_forge.shared import add_supported_preprocessor
+from modules_forge.utils import numpy_to_pytorch
+from modules_forge.shared import add_supported_control_model
+from modules_forge.supported_controlnet import ControlModelPatcher
+from lib_ipadapter.IPAdapterPlus import IPAdapterApply, InsightFaceLoader
+from pathlib import Path
+
+
+opIPAdapterApply = IPAdapterApply().apply_ipadapter
+opInsightFaceLoader = InsightFaceLoader().load_insight_face
+
+
+class PreprocessorClipVisionForIPAdapter(PreprocessorClipVision):
+    def __init__(self, name, url, filename):
+        super().__init__(name, url, filename)
+        self.tags = ['IP-Adapter']
+        self.model_filename_filters = ['IP-Adapter', 'IP_Adapter']
+        self.sorting_priority = 20
+
+    def __call__(self, input_image, resolution, slider_1=None, slider_2=None, slider_3=None, **kwargs):
+        cond = dict(
+            clip_vision=self.load_clipvision(),
+            image=numpy_to_pytorch(input_image),
+            weight_type="original",
+            noise=0.0,
+            embeds=None,
+            unfold_batch=False,
+        )
+        return cond
+
+
+class PreprocessorClipVisionWithInsightFaceForIPAdapter(PreprocessorClipVisionForIPAdapter):
+    def __init__(self, name, url, filename):
+        super().__init__(name, url, filename)
+        self.cached_insightface = None
+
+    def load_insightface(self):
+        if self.cached_insightface is None:
+            self.cached_insightface = opInsightFaceLoader()[0]
+        return self.cached_insightface
+
+    def __call__(self, input_image, resolution, slider_1=None, slider_2=None, slider_3=None, **kwargs):
+        cond = dict(
+            clip_vision=self.load_clipvision(),
+            insightface=self.load_insightface(),
+            image=numpy_to_pytorch(input_image),
+            weight_type="original",
+            noise=0.0,
+            embeds=None,
+            unfold_batch=False,
+        )
+        return cond
+
+
+class PreprocessorInsightFaceForInstantID(Preprocessor):
+    def __init__(self, name):
+        super().__init__()
+        self.name = name
+        self.tags = ['Instant-ID']
+        self.model_filename_filters = ['Instant-ID', 'Instant_ID']
+        self.sorting_priority = 20
+        self.slider_resolution = PreprocessorParameter(visible=False)
+        self.corp_image_with_a1111_mask_when_in_img2img_inpaint_tab = False
+        self.show_control_mode = False
+        self.sorting_priority = 10
+        self.cached_insightface = None
+
+    def load_insightface(self):
+        if self.cached_insightface is None:
+            self.cached_insightface = opInsightFaceLoader(name='antelopev2')[0]
+        return self.cached_insightface
+
+    def __call__(self, input_image, resolution, slider_1=None, slider_2=None, slider_3=None, **kwargs):
+        cond = dict(
+            clip_vision=None,
+            insightface=self.load_insightface(),
+            image=numpy_to_pytorch(input_image),
+            weight_type="original",
+            noise=0.0,
+            embeds=None,
+            unfold_batch=False,
+            instant_id=True
+        )
+        return cond
+
+
+add_supported_preprocessor(PreprocessorClipVisionForIPAdapter(
+    name='CLIP-ViT-H (IPAdapter)',
+    url='https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/model.safetensors',
+    filename='CLIP-ViT-H-14.safetensors'
+))
+
+add_supported_preprocessor(PreprocessorClipVisionForIPAdapter(
+    name='CLIP-ViT-bigG (IPAdapter)',
+    url='https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/image_encoder/model.safetensors',
+    filename='CLIP-ViT-bigG.safetensors'
+))
+
+add_supported_preprocessor(PreprocessorClipVisionWithInsightFaceForIPAdapter(
+    name='InsightFace+CLIP-H (IPAdapter)',
+    url='https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/model.safetensors',
+    filename='CLIP-ViT-H-14.safetensors'
+))
+
+add_supported_preprocessor(PreprocessorInsightFaceForInstantID(
+    name='InsightFace (InstantID)',
+))
+
+
+class IPAdapterPatcher(ControlModelPatcher):
+    @staticmethod
+    def try_build_from_state_dict(state_dict, ckpt_path):
+        model = state_dict
+
+        if ckpt_path.lower().endswith(".safetensors"):
+            st_model = {"image_proj": {}, "ip_adapter": {}}
+            for key in model.keys():
+                if key.startswith("image_proj."):
+                    st_model["image_proj"][key.replace("image_proj.", "")] = model[key]
+                elif key.startswith("ip_adapter."):
+                    st_model["ip_adapter"][key.replace("ip_adapter.", "")] = model[key]
+            model = st_model
+
+        if "ip_adapter" not in model.keys() or len(model["ip_adapter"]) == 0:
+            return None
+
+        o = IPAdapterPatcher(model)
+
+        model_filename = Path(ckpt_path).name.lower()
+        if 'v2' in model_filename:
+            o.faceid_v2 = True
+            o.weight_v2 = True
+
+        return o
+
+    def __init__(self, state_dict):
+        super().__init__()
+        self.ip_adapter = state_dict
+        self.faceid_v2 = False
+        self.weight_v2 = False
+        return
+
+    def process_before_every_sampling(self, process, cond, mask, *args, **kwargs):
+        unet = process.sd_model.forge_objects.unet
+
+        unet = opIPAdapterApply(
+            ipadapter=self.ip_adapter,
+            model=unet,
+            weight=self.strength,
+            start_at=self.start_percent,
+            end_at=self.end_percent,
+            faceid_v2=self.faceid_v2,
+            weight_v2=self.weight_v2,
+            attn_mask=mask.squeeze(1) if mask is not None else None,
+            **cond,
+        )[0]
+
+        process.sd_model.forge_objects.unet = unet
+        return
+
+
+add_supported_control_model(IPAdapterPatcher)