Fix parsing for real this time

Fixes #48 (again)

javascript/tagAutocomplete.js

@@ -92,15 +92,24 @@ function parseCSV(str) {
 
 // Load file
 function readFile(filePath) {
-    let request = new XMLHttpRequest();
-    request.open("GET", filePath, false);
-    request.send(null);
-    return request.responseText;
+    return new Promise(function (resolve, reject) {
+        let request = new XMLHttpRequest();
+        request.open("GET", filePath, true);
+        request.onload = function () {
+            var status = request.status;
+            if (status == 200) {
+                resolve(request.responseText);
+            } else {
+                reject(status);
+            }
+        };
+        request.send(null);
+    });
 }
 
 // Load CSV
-function loadCSV(path) {
-    let text = readFile(path);
+async function loadCSV(path) {
+    let text = await readFile(path);
     return parseCSV(text);
 }
 
@@ -220,6 +229,7 @@ function escapeRegExp(string) {
     return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
 }
 
+const TAG_REGEX = /[([]([^,()[\]:| ]+)(?::(?:\d+(?:\.\d+)?|\.\d+))?[)\]]|((?:\b|<)[^,|\n\r ]*(?:>|\b)?)/g
 let hideBlocked = false;
 
 // On click, insert the tag into the prompt textbox with respect to the cursor position
@@ -260,7 +270,7 @@ function insertTextAtCursor(textArea, result, tagword) {
 
     var optionalComma = "";
     if (tagType !== "wildcardFile") {
-        optionalComma = surrounding.match(new RegExp(escapeRegExp(`${tagword},`), "i")) !== null ? "" : ", ";
+        optionalComma = surrounding.match(new RegExp(`${escapeRegExp(tagword)}[,:]`, "i")) !== null ? "" : ", ";
     }
 
     // Replace partial tag word with new text, add comma if needed
@@ -277,7 +287,9 @@ function insertTextAtCursor(textArea, result, tagword) {
     textArea.dispatchEvent(new Event("input", { bubbles: true }));
 
     // Update previous tags with the edited prompt to prevent re-searching the same term
-    let tags = newPrompt.match(/[^,\n\r ]+/g);
+    let tags = [...newPrompt.matchAll(TAG_REGEX)]
+        .filter(x => x[2] !== undefined ? x[2].length > 0 : x)
+        .flatMap(x => x[1] || x[2])
     previousTags = tags;
 
     // Hide results after inserting
@@ -372,7 +384,7 @@ var allTags = [];
 var results = [];
 var tagword = "";
 var resultCount = 0;
-function autocomplete(textArea, prompt, fixedTag = null) {
+async function autocomplete(textArea, prompt, fixedTag = null) {
     // Return if the function is deactivated in the UI
     if (!acActive) return;
 
@@ -384,7 +396,10 @@ function autocomplete(textArea, prompt, fixedTag = null) {
 
     if (fixedTag === null) {
         // Match tags with RegEx to get the last edited one
-        let tags = prompt.match(/[^,\n\r ]+/g);
+        // We also match for the weighting format (e.g. "tag:1.0") here, and flatMap it to just the tag part if it exists
+        let tags = [...prompt.matchAll(TAG_REGEX)]
+            .filter(x => x[2] !== undefined ? x[2].length > 0 : x)
+            .flatMap(x => x[1] || x[2])
         let diff = difference(tags, previousTags)
         previousTags = tags;
 
@@ -405,7 +420,7 @@ function autocomplete(textArea, prompt, fixedTag = null) {
         tagword = fixedTag;
     }
 
-    tagword = tagword.toLowerCase().trim();
+    tagword = tagword.toLowerCase().replace(/[\n\r]/g, "");
 
     if (acConfig.useWildcards && [...tagword.matchAll(/\b__([^, ]+)__([^, ]*)\b/g)].length > 0) {
         // Show wildcards from a file with that name
@@ -421,7 +436,7 @@ function autocomplete(textArea, prompt, fixedTag = null) {
         else // Look in extensions wildcard files
             wcPair = wildcardExtFiles.find(x => x[1].toLowerCase() === wcFile);
 
-        let wildcards = readFile(`file/${wcPair[0]}/${wcPair[1]}.txt`).split("\n")
+        let wildcards = (await readFile(`file/${wcPair[0]}/${wcPair[1]}.txt`)).split("\n")
             .filter(x => x.trim().length > 0); // Remove empty lines
 
         results = wildcards.filter(x => (wcWord !== null && wcWord.length > 0) ? x.toLowerCase().includes(wcWord) : x) // Filter by tagword
@@ -560,14 +575,14 @@ function navigateInList(textArea, event) {
 }
 
 var styleAdded = false;
-onUiUpdate(function () {
+onUiUpdate(async function () {
     // Get our tag base path from the temp file
-    let tagBasePath = readFile("file/tmp/tagAutocompletePath.txt");
+    let tagBasePath = await readFile("file/tmp/tagAutocompletePath.txt");
 
     // Load config
     if (acConfig === null) {
         try {
-            acConfig = JSON.parse(readFile(`file/${tagBasePath}/config.json`));
+            acConfig = JSON.parse(await readFile(`file/${tagBasePath}/config.json`));
             if (acConfig.translation.onlyShowTranslation) {
                 acConfig.translation.searchByTranslation = true; // if only show translation, enable search by translation is necessary
             }
@@ -579,14 +594,14 @@ onUiUpdate(function () {
     // Load main tags and translations
     if (allTags.length === 0) {
         try {
-            allTags = loadCSV(`file/${tagBasePath}/${acConfig.tagFile}`);
+            allTags = await loadCSV(`file/${tagBasePath}/${acConfig.tagFile}`);
         } catch (e) {
             console.error("Error loading tags file: " + e);
             return;
         }
         if (acConfig.extra.extraFile) {
             try {
-                extras = loadCSV(`file/${tagBasePath}/${acConfig.extra.extraFile}`);
+                extras = await loadCSV(`file/${tagBasePath}/${acConfig.extra.extraFile}`);
                 if (acConfig.extra.onlyTranslationExtraFile) {
                     // This works purely on index, so it's not very robust. But a lot faster.
                     for (let i = 0, n = extras.length; i < n; i++) {
@@ -615,14 +630,14 @@ onUiUpdate(function () {
     // Load wildcards
     if (wildcardFiles.length === 0 && acConfig.useWildcards) {
         try {
-            let wcFileArr = readFile(`file/${tagBasePath}/temp/wc.txt`).split("\n");
+            let wcFileArr = (await readFile(`file/${tagBasePath}/temp/wc.txt`)).split("\n");
             let wcBasePath = wcFileArr[0].trim(); // First line should be the base path
             wildcardFiles = wcFileArr.slice(1)
                 .filter(x => x.trim().length > 0) // Remove empty lines
                 .map(x => [wcBasePath, x.trim().replace(".txt", "")]); // Remove file extension & newlines
 
             // To support multiple sources, we need to separate them using the provided "-----" strings
-            let wcExtFileArr = readFile(`file/${tagBasePath}/temp/wce.txt`).split("\n");
+            let wcExtFileArr = (await readFile(`file/${tagBasePath}/temp/wce.txt`)).split("\n");
             let splitIndices = [];
             for (let index = 0; index < wcExtFileArr.length; index++) {
                 if (wcExtFileArr[index].trim() === "-----") {
@@ -651,7 +666,7 @@ onUiUpdate(function () {
     // Load embeddings
     if (embeddings.length === 0 && acConfig.useEmbeddings) {
         try {
-            embeddings = readFile(`file/${tagBasePath}/temp/emb.txt`).split("\n")
+            embeddings = (await readFile(`file/${tagBasePath}/temp/emb.txt`)).split("\n")
                 .filter(x => x.trim().length > 0) // Remove empty lines
                 .map(x => x.replace(".bin", "").replace(".pt", "").replace(".png", "")); // Remove file extensions
         } catch (e) {

scripts/tag_autocomplete_helper.py

@@ -30,7 +30,7 @@ EMB_PATH = FILE_DIR.joinpath('embeddings')
 
 def find_ext_wildcard_paths():
     """Returns the path to the extension wildcards folder"""
-    found = list(EXT_PATH.rglob('**/wildcards/'))
+    found = list(EXT_PATH.glob('*/wildcards/'))
     return found
 
 
@@ -38,7 +38,8 @@ def find_ext_wildcard_paths():
 WILDCARD_EXT_PATHS = find_ext_wildcard_paths()
 
 # The path to the temporary files
-TEMP_PATH = TAGS_PATH.joinpath('temp')
+STATIC_TEMP_PATH = FILE_DIR.joinpath('tmp') # In the webui root, on windows it exists by default, on linux it doesn't
+TEMP_PATH = TAGS_PATH.joinpath('temp') # Extension specific temp files
 
 
 def get_wildcards():
@@ -55,8 +56,7 @@ def get_ext_wildcards():
 
     for path in WILDCARD_EXT_PATHS:
         wildcard_files.append(path.relative_to(FILE_DIR).as_posix())
-        wildcard_files.extend(p.relative_to(path).as_posix() for p in path.rglob(
-            "*.txt") if p.name != "put wildcards here.txt")
+        wildcard_files.extend(p.relative_to(path).as_posix() for p in path.rglob("*.txt") if p.name != "put wildcards here.txt")
         wildcard_files.append("-----")
 
     return wildcard_files
@@ -69,7 +69,7 @@ def get_embeddings():
 
 def write_tag_base_path():
     """Writes the tag base path to a fixed location temporary file"""
-    with open(FILE_DIR.joinpath('tmp/tagAutocompletePath.txt'), 'w', encoding="utf-8") as f:
+    with open(STATIC_TEMP_PATH.joinpath('tagAutocompletePath.txt'), 'w', encoding="utf-8") as f:
         f.write(TAGS_PATH.relative_to(FILE_DIR).as_posix())
 
 
@@ -81,6 +81,9 @@ def write_to_temp_file(name, data):
 
 # Write the tag base path to a fixed location temporary file
 # to enable the javascript side to find our files regardless of extension folder name
+if not STATIC_TEMP_PATH.exists():
+    STATIC_TEMP_PATH.mkdir(exist_ok=True)
+
 write_tag_base_path()
 
 # Check if the temp path exists and create it if not
@@ -95,8 +98,7 @@ write_to_temp_file('emb.txt', [])
 
 # Write wildcards to wc.txt if found
 if WILDCARD_PATH.exists():
-    wildcards = [WILDCARD_PATH.relative_to(
-        FILE_DIR).as_posix()] + get_wildcards()
+    wildcards = [WILDCARD_PATH.relative_to(FILE_DIR).as_posix()] + get_wildcards()
     if wildcards:
         write_to_temp_file('wc.txt', wildcards)