Merge pull request #52 from Kinsmir/main

Closes #49

javascript/tagAutocomplete.js

@@ -1,5 +1,6 @@
 var acConfig = null;
 var acActive = true;
+var acAppendComma = false;
 
 // Style for new elements. Gets appended to the Gradio root.
 let autocompleteCSS_dark = `
@@ -92,15 +93,24 @@ function parseCSV(str) {
 
 // Load file
 function readFile(filePath) {
-    let request = new XMLHttpRequest();
-    request.open("GET", filePath, false);
-    request.send(null);
-    return request.responseText;
+    return new Promise(function (resolve, reject) {
+        let request = new XMLHttpRequest();
+        request.open("GET", filePath, true);
+        request.onload = function () {
+            var status = request.status;
+            if (status == 200) {
+                resolve(request.responseText);
+            } else {
+                reject(status);
+            }
+        };
+        request.send(null);
+    });
 }
 
 // Load CSV
-function loadCSV(path) {
-    let text = readFile(path);
+async function loadCSV(path) {
+    let text = await readFile(path);
     return parseCSV(text);
 }
 
@@ -176,7 +186,7 @@ function createResultsDiv(textArea) {
 }
 
 // Create the checkbox to enable/disable autocomplete
-function createCheckbox() {
+function createCheckbox(text) {
     let label = document.createElement("label");
     let input = document.createElement("input");
     let span = document.createElement("span");
@@ -187,7 +197,7 @@ function createCheckbox() {
     input.setAttribute('class', 'gr-check-radio gr-checkbox')
     span.setAttribute('class', 'ml-2');
 
-    span.textContent = "Enable Autocomplete";
+    span.textContent = text;
 
     label.appendChild(input);
     label.appendChild(span);
@@ -220,6 +230,8 @@ function escapeRegExp(string) {
     return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
 }
 
+const WEIGHT_REGEX = /[([]([^,()[\]:| ]+)(?::(?:\d+(?:\.\d+)?|\.\d+))?[)\]]/g;
+const TAG_REGEX = /([^\s,|]+)/g
 let hideBlocked = false;
 
 // On click, insert the tag into the prompt textbox with respect to the cursor position
@@ -259,8 +271,8 @@ function insertTextAtCursor(textArea, result, tagword) {
     let afterInsertCursorPos = editStart + match.index + sanitizedText.length;
 
     var optionalComma = "";
-    if (tagType !== "wildcardFile") {
-        optionalComma = surrounding.match(new RegExp(escapeRegExp(`${tagword},`), "i")) !== null ? "" : ", ";
+    if (acAppendComma && tagType !== "wildcardFile") {
+        optionalComma = surrounding.match(new RegExp(`${escapeRegExp(tagword)}[,:]`, "i")) !== null ? "" : ", ";
     }
 
     // Replace partial tag word with new text, add comma if needed
@@ -277,7 +289,12 @@ function insertTextAtCursor(textArea, result, tagword) {
     textArea.dispatchEvent(new Event("input", { bubbles: true }));
 
     // Update previous tags with the edited prompt to prevent re-searching the same term
-    let tags = newPrompt.match(/[^, ]+/g);
+    let weightedTags = newPrompt.match(WEIGHT_REGEX)
+    let tags = newPrompt.match(TAG_REGEX)
+    if (weightedTags !== null) {
+        tags = tags.filter(tag => !weightedTags.some(weighted => tag.includes(weighted)))
+            .concat(weightedTags);
+    }
     previousTags = tags;
 
     // Hide results after inserting
@@ -372,7 +389,7 @@ var allTags = [];
 var results = [];
 var tagword = "";
 var resultCount = 0;
-function autocomplete(textArea, prompt, fixedTag = null) {
+async function autocomplete(textArea, prompt, fixedTag = null) {
     // Return if the function is deactivated in the UI
     if (!acActive) return;
 
@@ -384,12 +401,20 @@ function autocomplete(textArea, prompt, fixedTag = null) {
 
     if (fixedTag === null) {
         // Match tags with RegEx to get the last edited one
-        let tags = prompt.match(/[^, ]+/g);
-        let diff = difference(tags, previousTags)
+        // We also match for the weighting format (e.g. "tag:1.0") here, and combine the two to get the full tag word set
+        let weightedTags = prompt.match(WEIGHT_REGEX)
+        let tags = prompt.match(TAG_REGEX)
+        if (weightedTags !== null) {
+            tags = tags.filter(tag => !weightedTags.some(weighted => tag.includes(weighted)))
+                .concat(weightedTags);
+        }
+
+        let tagCountChange = tags.length - previousTags.length;
+        let diff = difference(tags, previousTags);
         previousTags = tags;
 
-        // Guard for no difference / only whitespace remaining
-        if (diff === null || diff.length === 0) {
+        // Guard for no difference / only whitespace remaining / last edited tag was fully removed
+        if (diff === null || diff.length === 0 || (diff.length === 1 && tagCountChange < 0)) {
             if (!hideBlocked) hideResults(textArea);
             return;
         }
@@ -405,7 +430,7 @@ function autocomplete(textArea, prompt, fixedTag = null) {
         tagword = fixedTag;
     }
 
-    tagword = tagword.toLowerCase();
+    tagword = tagword.toLowerCase().replace(/[\n\r]/g, "");
 
     if (acConfig.useWildcards && [...tagword.matchAll(/\b__([^, ]+)__([^, ]*)\b/g)].length > 0) {
         // Show wildcards from a file with that name
@@ -421,8 +446,8 @@ function autocomplete(textArea, prompt, fixedTag = null) {
         else // Look in extensions wildcard files
             wcPair = wildcardExtFiles.find(x => x[1].toLowerCase() === wcFile);
 
-        let wildcards = readFile(`file/${wcPair[0]}/${wcPair[1]}.txt`).split("\n")
-                        .filter(x => x.trim().length > 0); // Remove empty lines
+        let wildcards = (await readFile(`file/${wcPair[0]}/${wcPair[1]}.txt`)).split("\n")
+            .filter(x => x.trim().length > 0 && !x.startsWith('#'));  // Remove empty lines and comments
 
         results = wildcards.filter(x => (wcWord !== null && wcWord.length > 0) ? x.toLowerCase().includes(wcWord) : x) // Filter by tagword
             .map(x => [wcFile + ": " + x.trim(), "wildcardTag"]); // Mark as wildcard
@@ -560,14 +585,14 @@ function navigateInList(textArea, event) {
 }
 
 var styleAdded = false;
-onUiUpdate(function () {
+onUiUpdate(async function () {
     // Get our tag base path from the temp file
-    let tagBasePath = readFile("file/tmp/tagAutocompletePath.txt");
+    let tagBasePath = await readFile("file/tmp/tagAutocompletePath.txt");
 
     // Load config
     if (acConfig === null) {
         try {
-            acConfig = JSON.parse(readFile(`file/${tagBasePath}/config.json`));
+            acConfig = JSON.parse(await readFile(`file/${tagBasePath}/config.json`));
             if (acConfig.translation.onlyShowTranslation) {
                 acConfig.translation.searchByTranslation = true; // if only show translation, enable search by translation is necessary
             }
@@ -579,14 +604,14 @@ onUiUpdate(function () {
     // Load main tags and translations
     if (allTags.length === 0) {
         try {
-            allTags = loadCSV(`file/${tagBasePath}/${acConfig.tagFile}`);
+            allTags = await loadCSV(`file/${tagBasePath}/${acConfig.tagFile}`);
         } catch (e) {
             console.error("Error loading tags file: " + e);
             return;
         }
         if (acConfig.extra.extraFile) {
             try {
-                extras = loadCSV(`file/${tagBasePath}/${acConfig.extra.extraFile}`);
+                extras = await loadCSV(`file/${tagBasePath}/${acConfig.extra.extraFile}`);
                 if (acConfig.extra.onlyTranslationExtraFile) {
                     // This works purely on index, so it's not very robust. But a lot faster.
                     for (let i = 0, n = extras.length; i < n; i++) {
@@ -615,14 +640,14 @@ onUiUpdate(function () {
     // Load wildcards
     if (wildcardFiles.length === 0 && acConfig.useWildcards) {
         try {
-            let wcFileArr = readFile(`file/${tagBasePath}/temp/wc.txt`).split("\n");
+            let wcFileArr = (await readFile(`file/${tagBasePath}/temp/wc.txt`)).split("\n");
             let wcBasePath = wcFileArr[0].trim(); // First line should be the base path
             wildcardFiles = wcFileArr.slice(1)
                 .filter(x => x.trim().length > 0) // Remove empty lines
                 .map(x => [wcBasePath, x.trim().replace(".txt", "")]); // Remove file extension & newlines
-            
+
             // To support multiple sources, we need to separate them using the provided "-----" strings
-            let wcExtFileArr = readFile(`file/${tagBasePath}/temp/wce.txt`).split("\n");
+            let wcExtFileArr = (await readFile(`file/${tagBasePath}/temp/wce.txt`)).split("\n");
             let splitIndices = [];
             for (let index = 0; index < wcExtFileArr.length; index++) {
                 if (wcExtFileArr[index].trim() === "-----") {
@@ -640,7 +665,7 @@ onUiUpdate(function () {
                 wcExtFile = wcExtFile.slice(1)
                     .filter(x => x.trim().length > 0) // Remove empty lines
                     .map(x => x.trim().replace(base, "").replace(".txt", "")); // Remove file extension & newlines;
-                
+
                 wcExtFile = wcExtFile.map(x => [base, x]);
                 wildcardExtFiles.push(...wcExtFile);
             }
@@ -651,7 +676,7 @@ onUiUpdate(function () {
     // Load embeddings
     if (embeddings.length === 0 && acConfig.useEmbeddings) {
         try {
-            embeddings = readFile(`file/${tagBasePath}/temp/emb.txt`).split("\n")
+            embeddings = (await readFile(`file/${tagBasePath}/temp/emb.txt`)).split("\n")
                 .filter(x => x.trim().length > 0) // Remove empty lines
                 .map(x => x.replace(".bin", "").replace(".pt", "").replace(".png", "")); // Remove file extensions
         } catch (e) {
@@ -709,14 +734,41 @@ onUiUpdate(function () {
         }
     });
 
-    if (gradioApp().querySelector("#acActiveCheckbox") === null) {
+    // Add our custom options elements
+    if (gradioApp().querySelector("#tagAutocompleteOptions") === null) {
+        let optionsDiv = document.createElement("div");
+        optionsDiv.id = "tagAutocompleteOptions";
+        optionsDiv.classList.add("flex", "flex-col", "p-1", "px-1", "relative",  "text-sm");
+
+        let optionsInner = document.createElement("div");
+        optionsInner.classList.add("flex", "flex-row", "p-1", "gap-4", "text-gray-700");
+
+        // Add label
+        let title = document.createElement("p");
+        title.textContent = "Autocomplete options";
+        optionsDiv.appendChild(title);
+
         // Add toggle switch
-        let cb = createCheckbox();
-        cb.querySelector("input").checked = acActive;
-        cb.querySelector("input").addEventListener("change", (e) => {
+        let cbActive = createCheckbox("Enable Autocomplete");
+        cbActive.querySelector("input").checked = acActive;
+        cbActive.querySelector("input").addEventListener("change", (e) => {
             acActive = e.target.checked;
         });
-        quicksettings.parentNode.insertBefore(cb, quicksettings.nextSibling);
+        // Add comma switch
+        let cbComma = createCheckbox("Append commas");
+        acAppendComma = acConfig.appendComma;
+        cbComma.querySelector("input").checked = acAppendComma;
+        cbComma.querySelector("input").addEventListener("change", (e) => {
+            acAppendComma = e.target.checked;
+        });
+
+        // Add options to optionsDiv
+        optionsInner.appendChild(cbActive);
+        optionsInner.appendChild(cbComma);
+        optionsDiv.appendChild(optionsInner);
+
+        // Add options div to DOM
+        quicksettings.parentNode.insertBefore(optionsDiv, quicksettings.nextSibling);
     }
 
     if (styleAdded) return;
@@ -731,4 +783,4 @@ onUiUpdate(function () {
     }
     gradioApp().appendChild(acStyle);
     styleAdded = true;
-});
+});

scripts/tag_autocomplete_helper.py

@@ -11,6 +11,8 @@ FILE_DIR = Path().absolute()
 EXT_PATH = FILE_DIR.joinpath('extensions')
 
 # Tags base path
+
+
 def get_tags_base_path():
     script_path = Path(scripts.basedir())
     if (script_path.is_relative_to(EXT_PATH)):
@@ -18,29 +20,36 @@ def get_tags_base_path():
     else:
         return FILE_DIR.joinpath('tags')
 
+
 TAGS_PATH = get_tags_base_path()
 
 # The path to the folder containing the wildcards and embeddings
 WILDCARD_PATH = FILE_DIR.joinpath('scripts/wildcards')
 EMB_PATH = FILE_DIR.joinpath('embeddings')
 
+
 def find_ext_wildcard_paths():
     """Returns the path to the extension wildcards folder"""
-    found = list(EXT_PATH.rglob('**/wildcards/'))
+    found = list(EXT_PATH.glob('*/wildcards/'))
     return found
 
+
 # The path to the extension wildcards folder
 WILDCARD_EXT_PATHS = find_ext_wildcard_paths()
 
 # The path to the temporary files
-TEMP_PATH = TAGS_PATH.joinpath('temp')
+STATIC_TEMP_PATH = FILE_DIR.joinpath('tmp') # In the webui root, on windows it exists by default, on linux it doesn't
+TEMP_PATH = TAGS_PATH.joinpath('temp') # Extension specific temp files
+
 
 def get_wildcards():
     """Returns a list of all wildcards. Works on nested folders."""
     wildcard_files = list(WILDCARD_PATH.rglob("*.txt"))
-    resolved = [w.relative_to(WILDCARD_PATH).as_posix() for w in wildcard_files if w.name != "put wildcards here.txt"]
+    resolved = [w.relative_to(WILDCARD_PATH).as_posix(
+    ) for w in wildcard_files if w.name != "put wildcards here.txt"]
     return resolved
 
+
 def get_ext_wildcards():
     """Returns a list of all extension wildcards. Works on nested folders."""
     wildcard_files = []
@@ -57,11 +66,13 @@ def get_embeddings():
     """Returns a list of all embeddings"""
     return [str(e.relative_to(EMB_PATH)) for e in EMB_PATH.glob("**/*") if e.suffix in {".bin", ".pt", ".png"}]
 
+
 def write_tag_base_path():
     """Writes the tag base path to a fixed location temporary file"""
-    with open(FILE_DIR.joinpath('tmp/tagAutocompletePath.txt'), 'w', encoding="utf-8") as f:
+    with open(STATIC_TEMP_PATH.joinpath('tagAutocompletePath.txt'), 'w', encoding="utf-8") as f:
         f.write(TAGS_PATH.relative_to(FILE_DIR).as_posix())
 
+
 def write_to_temp_file(name, data):
     """Writes the given data to a temporary file"""
     with open(TEMP_PATH.joinpath(name), 'w', encoding="utf-8") as f:
@@ -70,6 +81,9 @@ def write_to_temp_file(name, data):
 
 # Write the tag base path to a fixed location temporary file
 # to enable the javascript side to find our files regardless of extension folder name
+if not STATIC_TEMP_PATH.exists():
+    STATIC_TEMP_PATH.mkdir(exist_ok=True)
+
 write_tag_base_path()
 
 # Check if the temp path exists and create it if not

tags/config.json

@@ -12,6 +12,7 @@
 	"useLeftRightArrowKeys": false,
 	"replaceUnderscores": true,
 	"escapeParentheses": true,
+	"appendComma": true,
 	"useWildcards": true,
 	"useEmbeddings": true,
 	"translation": {