Hotfix for old scripts/wildcards path loading

Fixes performance issues with many wildcard files, fixes #38

README.md

@@ -58,7 +58,9 @@ The config contains the following settings and defaults:
 		"negativePrompts": true
 	},
 	"maxResults": 5,
+	"resultStepLength": 500,
 	"showAllResults": false,
+	"useLeftRightArrowKeys": false,
 	"replaceUnderscores": true,
 	"escapeParentheses": true,
 	"useWildcards": true,
@@ -98,7 +100,9 @@ The config contains the following settings and defaults:
 | tagFile | Specifies the tag file to use. You can provide a custom tag database of your liking, but since the script was developed with Danbooru tags in mind, it might not work properly with other configurations.|
 | activeIn | Allows to selectively (de)activate the script for txt2img, img2img, and the negative prompts for both. |
 | maxResults | How many results to show max. For the default tag set, the results are ordered by occurence count. For embeddings and wildcards it will show all results in a scrollable list. |
+| resultStepLength | Allows to load results in smaller batches of the specified size for better performance in long lists or if showAllResults is true. |
 | showAllResults | If true, will ignore maxResults and show all results in a scrollable list. **Warning:** can lag your browser for long lists. |
+| useLeftRightArrowKeys | If true, left and right arrows will select the first/last result in the popup instead of moving the cursor in the textbox. |
 | replaceUnderscores | If true, undescores are replaced with spaces on clicking a tag. Might work better for some models. |
 | escapeParentheses | If true, escapes tags containing () so they don't contribute to the web UI's prompt weighting functionality. |
 | useWildcards | Used to toggle the wildcard completion functionality. |
@@ -165,4 +169,4 @@ or of e621:
 |7	    | Meta        |
 |8	    | Lore        |
 
-The tag type is used for coloring entries in the result list.
+The tag type is used for coloring entries in the result list.

javascript/tagAutocomplete.js

@@ -134,19 +134,27 @@ function difference(a, b) {
 
 // Get the identifier for the text area to differentiate between positive and negative
 function getTextAreaIdentifier(textArea) {
+    let txt2img_p = gradioApp().querySelector('#txt2img_prompt > label > textarea');
     let txt2img_n = gradioApp().querySelector('#txt2img_neg_prompt > label > textarea');
-    let img2img = gradioApp().querySelector('#tab_img2img');
-    let img2img_p = img2img.querySelector('#img2img_prompt > label > textarea');
-    let img2img_n = img2img.querySelector('#img2img_neg_prompt > label > textarea');
+    let img2img_p = gradioApp().querySelector('#img2img_prompt > label > textarea');
+    let img2img_n = gradioApp().querySelector('#img2img_neg_prompt > label > textarea');
 
     let modifier = "";
-    if (textArea === img2img_p || textArea === img2img_n) {
-        modifier += ".img2img";
-    }
-    if (textArea === txt2img_n || textArea === img2img_n) {
-        modifier += ".n";
-    } else {
-        modifier += ".p";
+    switch (textArea) {
+        case txt2img_p:
+            modifier = ".txt2img.p";
+            break;
+        case txt2img_n:
+            modifier = ".txt2img.n";
+            break;
+        case img2img_p:
+            modifier = ".img2img.p";
+            break;
+        case img2img_n:
+            modifier = ".img2img.n";
+            break;
+        default:
+            break;
     }
     return modifier;
 }
@@ -198,13 +206,11 @@ function isVisible(textArea) {
 }
 function showResults(textArea) {
     let textAreaId = getTextAreaIdentifier(textArea);
-
     let resultsDiv = gradioApp().querySelector('.autocompleteResults' + textAreaId);
     resultsDiv.style.display = "block";
 }
 function hideResults(textArea) {
     let textAreaId = getTextAreaIdentifier(textArea);
-
     let resultsDiv = gradioApp().querySelector('.autocompleteResults' + textAreaId);
     resultsDiv.style.display = "none";
     selectedTag = null;
@@ -285,23 +291,26 @@ function insertTextAtCursor(textArea, result, tagword) {
     }
 }
 
-function addResultsToList(textArea, results, tagword) {
+function addResultsToList(textArea, results, tagword, resetList) {
     let textAreaId = getTextAreaIdentifier(textArea);
     let resultDiv = gradioApp().querySelector('.autocompleteResults' + textAreaId);
     let resultsList = resultDiv.querySelector('ul');
 
     // Reset list, selection and scrollTop since the list changed
-    resultsList.innerHTML = "";
-    selectedTag = null;
-    resultDiv.scrollTop = 0;
+    if (resetList) {
+        resultsList.innerHTML = "";
+        selectedTag = null;
+        resultDiv.scrollTop = 0;
+        resultCount = 0;
+    }
 
     // Find right colors from config
     let tagFileName = acConfig.tagFile.split(".")[0];
     let tagColors = acConfig.colors;
-
     let mode = gradioApp().querySelector('.dark') ? 0 : 1;
+    let nextLength = Math.min(results.length, resultCount + acConfig.resultStepLength);
 
-    for (let i = 0; i < results.length; i++) {
+    for (let i = resultCount; i < nextLength; i++) {
         let result = results[i];
         let li = document.createElement("li");
 
@@ -331,34 +340,38 @@ function addResultsToList(textArea, results, tagword) {
         // Add element to list
         resultsList.appendChild(li);
     }
+    resultCount = nextLength;
 }
 
-function updateSelectionStyle(textArea, num) {
+function updateSelectionStyle(textArea, newIndex, oldIndex) {
     let textAreaId = getTextAreaIdentifier(textArea);
     let resultDiv = gradioApp().querySelector('.autocompleteResults' + textAreaId);
     let resultsList = resultDiv.querySelector('ul');
     let items = resultsList.getElementsByTagName('li');
 
-    for (let i = 0; i < items.length; i++) {
-        items[i].classList.remove('selected');
+    if (oldIndex != null) {
+        items[oldIndex].classList.remove('selected');
     }
 
-    items[num].classList.add('selected');
+    // make it safer
+    if (newIndex !== null) {
+        items[newIndex].classList.add('selected');
+    }
 
     // Set scrolltop to selected item if we are showing more than max results
     if (items.length > acConfig.maxResults) {
-        let selected = items[num];
+        let selected = items[newIndex];
         resultDiv.scrollTop = selected.offsetTop - resultDiv.offsetTop;
     }
 }
 
-wildcardFiles = [];
-wildcards = {};
-embeddings = [];
-allTags = [];
-results = [];
-tagword = "";
-resultCount = 0;
+var wildcardFiles = [];
+var wildcardExtFiles = [];
+var embeddings = [];
+var allTags = [];
+var results = [];
+var tagword = "";
+var resultCount = 0;
 function autocomplete(textArea, prompt, fixedTag = null) {
     // Return if the function is deactivated in the UI
     if (!acActive) return;
@@ -394,20 +407,34 @@ function autocomplete(textArea, prompt, fixedTag = null) {
 
     tagword = tagword.toLowerCase();
 
-    if (acConfig.useWildcards && [...tagword.matchAll(/\b__([^,_ ]+)__([^, ]*)\b/g)].length > 0) {
+    if (acConfig.useWildcards && [...tagword.matchAll(/\b__([^, ]+)__([^, ]*)\b/g)].length > 0) {
         // Show wildcards from a file with that name
-        wcMatch = [...tagword.matchAll(/\b__([^,_ ]+)__([^, ]*)\b/g)]
+        wcMatch = [...tagword.matchAll(/\b__([^, ]+)__([^, ]*)\b/g)]
         let wcFile = wcMatch[0][1];
         let wcWord = wcMatch[0][2];
-        results = wildcards[wcFile].filter(x => (wcWord !== null) ? x.toLowerCase().includes(wcWord) : x) // Filter by tagword
+
+        let wcBasePath = "";
+        if (wildcardExtFiles.includes(wcFile))
+            wcBasePath = "extensions/wildcards/wildcards";
+        else if (wildcardFiles.includes(wcFile))
+            wcBasePath = "scripts/wildcards";
+        else
+            throw "No valid wildcard file found";
+
+        let wildcards = readFile(`file/${wcBasePath}/${wcFile}.txt`).split("\n")
+                        .filter(x => x.trim().length > 0); // Remove empty lines
+
+        results = wildcards.filter(x => (wcWord !== null && wcWord.length > 0) ? x.toLowerCase().includes(wcWord) : x) // Filter by tagword
             .map(x => [wcFile + ": " + x.trim(), "wildcardTag"]); // Mark as wildcard
     } else if (acConfig.useWildcards && (tagword.startsWith("__") && !tagword.endsWith("__") || tagword === "__")) {
         // Show available wildcard files
         let tempResults = [];
         if (tagword !== "__") {
-            tempResults = wildcardFiles.filter(x => x.toLowerCase().includes(tagword.replace("__", ""))) // Filter by tagword
+            let lmb = (x) => x.toLowerCase().includes(tagword.replace("__", ""))
+            tempResults = wildcardFiles.filter(lmb).concat(wildcardExtFiles.filter(lmb)) // Filter by tagword
+            
         } else {
-            tempResults = wildcardFiles;
+            tempResults = wildcardFiles.concat(wildcardExtFiles);
         }
         results = tempResults.map(x => ["Wildcards: " + x.trim(), "wildcardFile"]); // Mark as wildcard
     } else if (acConfig.useEmbeddings && tagword.match(/<[^,> ]*>?/g)) {
@@ -438,29 +465,31 @@ function autocomplete(textArea, prompt, fixedTag = null) {
         }
     }
 
-    resultCount = results.length;
-
     // Guard for empty results
-    if (resultCount === 0) {
+    if (!results.length) {
         hideResults(textArea);
         return;
     }
 
     showResults(textArea);
-    addResultsToList(textArea, results, tagword);
+    addResultsToList(textArea, results, tagword, true);
 }
 
 function navigateInList(textArea, event) {
     // Return if the function is deactivated in the UI
     if (!acActive) return;
 
-    validKeys = ["ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "Enter", "Escape"];
+    validKeys = ["ArrowUp", "ArrowDown", "PageUp", "PageDown", "Home", "End", "Enter", "Tab", "Escape"];
+    if (acConfig.useLeftRightArrowKeys)
+        validKeys.push("ArrowLeft", "ArrowRight");
 
     if (!validKeys.includes(event.key)) return;
     if (!isVisible(textArea)) return
     // Return if ctrl key is pressed to not interfere with weight editing shortcut
     if (event.ctrlKey || event.altKey) return;
 
+    oldSelectedTag = selectedTag;
+
     switch (event.key) {
         case "ArrowUp":
             if (selectedTag === null) {
@@ -476,6 +505,26 @@ function navigateInList(textArea, event) {
                 selectedTag = (selectedTag + 1) % resultCount;
             }
             break;
+        case "PageUp":
+            if (selectedTag === null || selectedTag === 0) {
+                selectedTag = resultCount - 1;
+            } else {
+                selectedTag = (Math.max(selectedTag - 5, 0) + resultCount) % resultCount;
+            }
+            break;
+        case "PageDown":
+            if (selectedTag === null || selectedTag === resultCount - 1) {
+                selectedTag = 0;
+            } else {
+                selectedTag = Math.min(selectedTag + 5, resultCount - 1) % resultCount;
+            }
+            break;
+        case "Home":
+            selectedTag = 0;
+            break;
+        case "End":
+            selectedTag = resultCount - 1;
+            break;
         case "ArrowLeft":
             selectedTag = 0;
             break;
@@ -487,20 +536,30 @@ function navigateInList(textArea, event) {
                 insertTextAtCursor(textArea, results[selectedTag], tagword);
             }
             break;
+        case "Tab":
+            if (selectedTag === null) {
+                selectedTag = 0;
+            }
+            insertTextAtCursor(textArea, results[selectedTag], tagword);
+            break;
         case "Escape":
             hideResults(textArea);
             break;
     }
+    if (selectedTag === resultCount - 1
+        && (event.key === "ArrowUp" || event.key === "ArrowDown" || event.key === "ArrowLeft" || event.key === "ArrowRight")) {
+        addResultsToList(textArea, results, tagword, false);
+    }
     // Update highlighting
     if (selectedTag !== null)
-        updateSelectionStyle(textArea, selectedTag);
+        updateSelectionStyle(textArea, selectedTag, oldSelectedTag);
 
     // Prevent default behavior
     event.preventDefault();
     event.stopPropagation();
 }
 
-styleAdded = false;
+var styleAdded = false;
 onUiUpdate(function () {
     // Load config
     if (acConfig === null) {
@@ -556,17 +615,11 @@ onUiUpdate(function () {
             wildcardFiles = readFile("file/tags/temp/wc.txt").split("\n")
                 .filter(x => x.trim().length > 0) // Remove empty lines
                 .map(x => x.trim().replace(".txt", "")); // Remove file extension & newlines
-
-            wildcardFiles.forEach(fName => {
-                try {
-                    wildcards[fName] = readFile(`file/scripts/wildcards/${fName}.txt`).split("\n")
-                        .filter(x => x.trim().length > 0); // Remove empty lines
-                } catch (e) {
-                    console.log(`Could not load wildcards for ${fName}`);
-                }
-            });
+            wildcardExtFiles = readFile("file/tags/temp/wce.txt").split("\n")
+                .filter(x => x.trim().length > 0) // Remove empty lines
+                .map(x => x.trim().replace(".txt", "")); // Remove file extension & newlines
         } catch (e) {
-            console.error("Error loading wildcardNames.txt: " + e);
+            console.error("Error loading wildcards: " + e);
         }
     }
     // Load embeddings
@@ -574,7 +627,7 @@ onUiUpdate(function () {
         try {
             embeddings = readFile("file/tags/temp/emb.txt").split("\n")
                 .filter(x => x.trim().length > 0) // Remove empty lines
-                .map(x => x.replace(".bin", "").replace(".pt", "")); // Remove file extensions
+                .map(x => x.replace(".bin", "").replace(".pt", "").replace(".png", "")); // Remove file extensions
         } catch (e) {
             console.error("Error loading embeddings.txt: " + e);
         }
@@ -591,25 +644,23 @@ onUiUpdate(function () {
 
     // Not found, we're on a page without prompt textareas
     if (textAreas.every(v => v === null || v === undefined)) return;
-    // Already added?
-    if (gradioApp().querySelector('.autocompleteResults.p') !== null
-        && (gradioApp().querySelector('.autocompleteResults.n') === null
-            && !acConfig.activeIn.negativePrompts)) {
+    // Already added or unnecessary to add
+    if (gradioApp().querySelector('.autocompleteResults.p')) {
+        if (gradioApp().querySelector('.autocompleteResults.n') || !acConfig.activeIn.negativePrompts) {
+            return;
+        }
+    } else if (!acConfig.activeIn.txt2img && !acConfig.activeIn.img2img) {
         return;
     }
 
     textAreas.forEach(area => {
-        // Skip directly if not found on the page
-        if (area === null || area === undefined) return;
 
         // Return if autocomplete is disabled for the current area type in config
         let textAreaId = getTextAreaIdentifier(area);
-        if (textAreaId.includes("p") || (textAreaId.includes("n") && acConfig.activeIn.negativePrompts)) {
-            if (textAreaId.includes("img2img")) {
-                if (!acConfig.activeIn.img2img) return;
-            } else {
-                if (!acConfig.activeIn.txt2img) return;
-            }
+        if ((!acConfig.activeIn.img2img && textAreaId.includes("img2img"))
+            || (!acConfig.activeIn.txt2img && textAreaId.includes("txt2img"))
+            || (!acConfig.activeIn.negativePrompts && textAreaId.includes("n"))) {
+            return;
         }
 
         // Only add listeners once

scripts/tag_autocomplete_helper.py

@@ -6,6 +6,7 @@ from pathlib import Path
 # The path to the folder containing the wildcards and embeddings
 FILE_DIR = Path().absolute()
 WILDCARD_PATH = FILE_DIR.joinpath('scripts/wildcards')
+WILDCARD_EXT_PATH = FILE_DIR.joinpath('extensions/wildcards/wildcards')
 EMB_PATH = FILE_DIR.joinpath('embeddings')
 # The path to the temporary file
 TEMP_PATH = FILE_DIR.joinpath('tags/temp')
@@ -14,13 +15,19 @@ TEMP_PATH = FILE_DIR.joinpath('tags/temp')
 def get_wildcards():
     """Returns a list of all wildcards. Works on nested folders."""
     wildcard_files = list(WILDCARD_PATH.rglob("*.txt"))
-    resolved = [str(w.relative_to(WILDCARD_PATH)) for w in wildcard_files]
+    resolved = [w.relative_to(WILDCARD_PATH).as_posix() for w in wildcard_files if w.name != "put wildcards here.txt"]
+    return resolved
+
+def get_ext_wildcards():
+    """Returns a list of all extension wildcards. Works on nested folders."""
+    wildcard_files = list(WILDCARD_EXT_PATH.rglob("*.txt"))
+    resolved = [w.relative_to(WILDCARD_EXT_PATH).as_posix() for w in wildcard_files if w.name != "put wildcards here.txt"]
     return resolved
 
 
 def get_embeddings():
     """Returns a list of all embeddings"""
-    return [str(e.relative_to(EMB_PATH)) for e in EMB_PATH.glob("**/*") if e.suffix in {".bin", ".pt"}]
+    return [str(e.relative_to(EMB_PATH)) for e in EMB_PATH.glob("**/*") if e.suffix in {".bin", ".pt", ".png"}]
 
 
 def write_to_temp_file(name, data):
@@ -32,10 +39,12 @@ def write_to_temp_file(name, data):
 # Check if the temp path exists and create it if not
 if not TEMP_PATH.exists():
     TEMP_PATH.mkdir(parents=True, exist_ok=True)
-    # Set up files to ensure the script doesn't fail to load them
-    # even if no wildcards or embeddings are found
-    write_to_temp_file('wc.txt', [])
-    write_to_temp_file('emb.txt', [])
+
+# Set up files to ensure the script doesn't fail to load them
+# even if no wildcards or embeddings are found
+write_to_temp_file('wc.txt', [])
+write_to_temp_file('wce.txt', [])
+write_to_temp_file('emb.txt', [])
 
 # Write wildcards to wc.txt if found
 if WILDCARD_PATH.exists():
@@ -43,6 +52,12 @@ if WILDCARD_PATH.exists():
     if wildcards:
         write_to_temp_file('wc.txt', wildcards)
 
+# Write extension wildcards to wce.txt if found
+if WILDCARD_EXT_PATH.exists():
+    wildcards_ext = get_ext_wildcards()
+    if wildcards_ext:
+        write_to_temp_file('wce.txt', wildcards_ext)
+
 # Write embeddings to emb.txt if found
 if EMB_PATH.exists():
     embeddings = get_embeddings()

tags/config.json

@@ -6,7 +6,9 @@
 		"negativePrompts": true
 	},
 	"maxResults": 5,
+	"resultStepLength": 500,
 	"showAllResults": false,
+	"useLeftRightArrowKeys": false,
 	"replaceUnderscores": true,
 	"escapeParentheses": true,
 	"useWildcards": true,