From 237f80681a9f22f8ab3c983cc12509ed1394503f Mon Sep 17 00:00:00 2001 From: lllyasviel Date: Sun, 11 Feb 2024 20:28:02 -0800 Subject: [PATCH] add more notes to clip codes --- ldm_patched/modules/sd1_clip.py | 51 +++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/ldm_patched/modules/sd1_clip.py b/ldm_patched/modules/sd1_clip.py index 8c77b739..50c67d6d 100644 --- a/ldm_patched/modules/sd1_clip.py +++ b/ldm_patched/modules/sd1_clip.py @@ -16,6 +16,9 @@ import ldm_patched.modules.clip_model import json from transformers import CLIPTextModel, CLIPTextConfig, modeling_utils + +# Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py +# This function is only for reference, and not used in the backend or runtime. def gen_empty_tokens(special_tokens, length): start_token = special_tokens.get("start", None) end_token = special_tokens.get("end", None) @@ -29,6 +32,9 @@ def gen_empty_tokens(special_tokens, length): return output class ClipTokenWeightEncoder: + + # Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py + # This function is only for reference, and not used in the backend or runtime. def encode_token_weights(self, token_weight_pairs): to_encode = list() max_token_len = 0 @@ -127,6 +133,8 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder): self.layer = self.layer_default[0] self.layer_idx = self.layer_default[1] + # Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py + # This function is only for reference, and not used in the backend or runtime. def set_up_textual_embeddings(self, tokens, current_embeds): out_tokens = [] next_new_token = token_dict_size = current_embeds.weight.shape[0] - 1 @@ -166,6 +174,8 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder): return processed_tokens + # Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py + # This function is only for reference, and not used in the backend or runtime. def forward(self, tokens): backup_embeds = self.transformer.get_input_embeddings() device = backup_embeds.weight.device @@ -214,6 +224,9 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder): self.text_projection[:] = sd.pop("text_projection.weight").transpose(0, 1) return self.transformer.load_state_dict(sd, strict=False) + +# Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py +# This function is only for reference, and not used in the backend or runtime. def parse_parentheses(string): result = [] current_item = "" @@ -242,6 +255,9 @@ def parse_parentheses(string): result.append(current_item) return result + +# Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py +# This function is only for reference, and not used in the backend or runtime. def token_weights(string, current_weight): a = parse_parentheses(string) out = [] @@ -262,16 +278,25 @@ def token_weights(string, current_weight): out += [(x, current_weight)] return out + +# Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py +# This function is only for reference, and not used in the backend or runtime. def escape_important(text): text = text.replace("\\)", "\0\1") text = text.replace("\\(", "\0\2") return text + +# Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py +# This function is only for reference, and not used in the backend or runtime. def unescape_important(text): text = text.replace("\0\1", ")") text = text.replace("\0\2", "(") return text + +# Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py +# This function is only for reference, and not used in the backend or runtime. def safe_load_embed_zip(embed_path): with zipfile.ZipFile(embed_path) as myzip: names = list(filter(lambda a: "data/" in a, myzip.namelist())) @@ -291,6 +316,9 @@ def safe_load_embed_zip(embed_path): del embed return out + +# Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py +# This function is only for reference, and not used in the backend or runtime. def expand_directory_list(directories): dirs = set() for x in directories: @@ -299,6 +327,9 @@ def expand_directory_list(directories): dirs.add(root) return list(dirs) + +# Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py +# This function is only for reference, and not used in the backend or runtime. def load_embed(embedding_name, embedding_directory, embedding_size, embed_key=None): if isinstance(embedding_directory, str): embedding_directory = [embedding_directory] @@ -398,6 +429,8 @@ class SDTokenizer: self.embedding_size = embedding_size self.embedding_key = embedding_key + # Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py + # This function is only for reference, and not used in the backend or runtime. def _try_get_embedding(self, embedding_name:str): ''' Takes a potential embedding name and tries to retrieve it. @@ -411,7 +444,8 @@ class SDTokenizer: return (embed, embedding_name[len(stripped):]) return (embed, "") - + # Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py + # This function is only for reference, and not used in the backend or runtime. def tokenize_with_weights(self, text:str, return_word_ids=False): ''' Takes a prompt and converts it to a list of (token, weight, word id) elements. @@ -494,7 +528,8 @@ class SDTokenizer: return batched_tokens - + # Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py + # This function is only for reference, and not used in the backend or runtime. def untokenize(self, token_weight_pair): return list(map(lambda a: (a, self.inv_vocab[a[0]]), token_weight_pair)) @@ -505,11 +540,15 @@ class SD1Tokenizer: self.clip = "clip_{}".format(self.clip_name) setattr(self, self.clip, tokenizer(embedding_directory=embedding_directory)) + # Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py + # This function is only for reference, and not used in the backend or runtime. def tokenize_with_weights(self, text:str, return_word_ids=False): out = {} out[self.clip_name] = getattr(self, self.clip).tokenize_with_weights(text, return_word_ids) return out + # Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py + # This function is only for reference, and not used in the backend or runtime. def untokenize(self, token_weight_pair): return getattr(self, self.clip).untokenize(token_weight_pair) @@ -521,16 +560,24 @@ class SD1ClipModel(torch.nn.Module): self.clip = "clip_{}".format(self.clip_name) setattr(self, self.clip, clip_model(device=device, dtype=dtype, **kwargs)) + # Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py + # This function is only for reference, and not used in the backend or runtime. def clip_layer(self, layer_idx): getattr(self, self.clip).clip_layer(layer_idx) + # Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py + # This function is only for reference, and not used in the backend or runtime. def reset_clip_layer(self): getattr(self, self.clip).reset_clip_layer() + # Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py + # This function is only for reference, and not used in the backend or runtime. def encode_token_weights(self, token_weight_pairs): token_weight_pairs = token_weight_pairs[self.clip_name] out, pooled = getattr(self, self.clip).encode_token_weights(token_weight_pairs) return out, pooled + # Taken from https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/sd1_clip.py + # This function is only for reference, and not used in the backend or runtime. def load_sd(self, sd): return getattr(self, self.clip).load_sd(sd)