From 97cbada88297591c17876303c78ab284d3c0b149 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Tue, 20 Dec 2022 13:28:43 +0100 Subject: [PATCH 1/5] Sort yaml tags by count --- scripts/tag_autocomplete_helper.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py index 79f19b1..cb9c71b 100644 --- a/scripts/tag_autocomplete_helper.py +++ b/scripts/tag_autocomplete_helper.py @@ -73,8 +73,10 @@ def get_ext_wildcard_tags(): wildcard_tags[tag] += 1 except yaml.YAMLError as exc: print(exc) + # Sort by count + sorted_tags = sorted(wildcard_tags.items(), key=lambda item: item[1], reverse=True) output = [] - for tag, count in wildcard_tags.items(): + for tag, count in sorted_tags: output.append(f"{tag},{count}") return output @@ -136,12 +138,10 @@ if WILDCARD_EXT_PATHS is not None: wildcards_ext = get_ext_wildcards() if wildcards_ext: write_to_temp_file('wce.txt', wildcards_ext) - -# Write extension wildcards to wce.txt if found -if WILDCARD_EXT_PATHS is not None: - wildcards_ext = get_ext_wildcard_tags() - if wildcards_ext: - write_to_temp_file('wcet.txt', wildcards_ext) + # Write yaml extension wildcards to wcet.txt if found + wildcards_yaml_ext = get_ext_wildcard_tags() + if wildcards_yaml_ext: + write_to_temp_file('wcet.txt', wildcards_yaml_ext) # Write embeddings to emb.txt if found if EMB_PATH.exists(): From f1a437ff4887d29f08df6795076d4648489b4e70 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Tue, 20 Dec 2022 17:08:09 +0100 Subject: [PATCH 2/5] Autocompletion for UMI yaml wildcards --- javascript/tagAutocomplete.js | 96 ++++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 19 deletions(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 0b8ca6e..5a21201 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -342,7 +342,7 @@ function escapeHTML(unsafeText) { } const WEIGHT_REGEX = /[([]([^,()[\]:| ]+)(?::(?:\d+(?:\.\d+)?|\.\d+))?[)\]]/g; -const TAG_REGEX = /([^\s,|]+)/g +const TAG_REGEX = /(<[^\t\n\r,>]+>?|[^\s,|<>]+|<)/g let hideBlocked = false; // On click, insert the tag into the prompt textbox with respect to the cursor position @@ -358,6 +358,8 @@ function insertTextAtCursor(textArea, result, tagword) { sanitizedText = "__" + text.replace("Wildcards: ", "") + "__"; } else if (tagType === "wildcardTag") { sanitizedText = text.replace(/^.*?: /g, ""); + } else if (tagType === "yamlWildcard" && !yamlWildcards.includes(text)) { + sanitizedText = text.replaceAll("_", " "); // Replace underscores only if the yaml tag is not using them } else if (tagType === "embedding") { sanitizedText = `<${text.replace(/^.*?: /g, "")}>`; } else { @@ -382,7 +384,7 @@ function insertTextAtCursor(textArea, result, tagword) { let afterInsertCursorPos = editStart + match.index + sanitizedText.length; var optionalComma = ""; - if (CFG.appendComma && tagType !== "wildcardFile") { + if (CFG.appendComma && tagType !== "wildcardFile" && tagType !== "yamlWildcard") { optionalComma = surrounding.match(new RegExp(`${escapeRegExp(tagword)}[,:]`, "i")) !== null ? "" : ", "; } @@ -409,6 +411,12 @@ function insertTextAtCursor(textArea, result, tagword) { } previousTags = tags; + // If it was a yaml wildcard, also update the umiPreviousTags + if (tagType === "yamlWildcard" && originalTagword.length > 0) { + let umiSubPrompt = originalTagword.match(/<[^\s]*?\[[^,<>]*[\]|]?>?/g)[0]; + umiPreviousTags = [...umiSubPrompt.matchAll(/(?:\[|\||--)([^<>\[\]\-|]+)/g)].map(x => x[1]); + } + // Hide results after inserting if (tagType === "wildcardFile") { // If it's a wildcard, we want to keep the results open so the user can select another wildcard @@ -490,18 +498,20 @@ function addResultsToList(textArea, results, tagword, resetList) { // Add post count & color if it's a tag // Wildcards & Embeds have no tag type if (!result[1].startsWith("wildcard") && result[1] !== "embedding") { - // Set the color of the tag - let tagType = result[1]; - let colorGroup = tagColors[tagFileName]; - // Default to danbooru scheme if no matching one is found - if (!colorGroup) - colorGroup = tagColors["danbooru"]; + if (!result[1].startsWith("yaml")) { + // Set the color of the tag + let tagType = result[1]; + let colorGroup = tagColors[tagFileName]; + // Default to danbooru scheme if no matching one is found + if (!colorGroup) + colorGroup = tagColors["danbooru"]; - // Set tag type to invalid if not found - if (!colorGroup[tagType]) - tagType = "-1"; + // Set tag type to invalid if not found + if (!colorGroup[tagType]) + tagType = "-1"; - itemText.style = `color: ${colorGroup[tagType][mode]};`; + itemText.style = `color: ${colorGroup[tagType][mode]};`; + } // Post count if (result[2] && !isNaN(result[2])) { @@ -555,9 +565,12 @@ function updateSelectionStyle(textArea, newIndex, oldIndex) { var wildcardFiles = []; var wildcardExtFiles = []; +var yamlWildcards = []; +var umiPreviousTags = []; var embeddings = []; var results = []; var tagword = ""; +var originalTagword = ""; var resultCount = 0; async function autocomplete(textArea, prompt, fixedTag = null) { // Return if the function is deactivated in the UI @@ -575,8 +588,8 @@ async function autocomplete(textArea, prompt, fixedTag = null) { let weightedTags = [...prompt.matchAll(WEIGHT_REGEX)] .map(match => match[1]); let tags = prompt.match(TAG_REGEX) - if (weightedTags !== null) { - tags = tags.filter(tag => !weightedTags.some(weighted => tag.includes(weighted))) + if (weightedTags !== null && tags !== null) { + tags = tags.filter(tag => !weightedTags.some(weighted => tag.includes(weighted) && !tag.startsWith("<["))) .concat(weightedTags); } @@ -632,6 +645,41 @@ async function autocomplete(textArea, prompt, fixedTag = null) { tempResults = wildcardFiles.concat(wildcardExtFiles); } results = tempResults.map(x => ["Wildcards: " + x[1].trim(), "wildcardFile"]); // Mark as wildcard + } else if (CFG.useWildcards && tagword.match(/<[^\s]*?\[[^,<>]*[\]|]?>?/g)) { + // We are in a UMI yaml tag definition, parse further + let umiSubPrompt = tagword.match(/<[^\s]*?\[[^,<>]*[\]|]?>?/g)[0]; + let umiTags = [...umiSubPrompt.matchAll(/(?:\[|\||--)([^<>\[\]\-|]+)/g)].map(x => x[1]); + if (umiTags.length > 0) { + // Get difference for subprompt + let diff = difference(umiTags, umiPreviousTags); + umiPreviousTags = umiTags; + + // Show all condition + let showAll = umiSubPrompt.endsWith("[") || umiSubPrompt.endsWith("[--") || umiSubPrompt.endsWith("|"); + + // Exit early if the user closed the bracket manually + if ((!diff || diff.length === 0) && (!showAll || umiSubPrompt.endsWith("]"))) { + if (!hideBlocked) hideResults(textArea); + return; + } + + let umiTagword = diff[0]; + let tempResults = []; + if (umiTagword && umiTagword.length > 0) { + originalTagword = tagword; + tagword = umiTagword; + + let searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(umiTagword)}`, 'i') + let baseFilter = x => x[0].toLowerCase().search(searchRegex) > -1; + let spaceIncludeFilter = x => x[0].toLowerCase().replaceAll(" ", "_").search(searchRegex) > -1; + tempResults = yamlWildcards.filter(x => baseFilter(x) || spaceIncludeFilter(x)) // Filter by tagword + results = tempResults.map(x => [x[0].trim(), "yamlWildcard", x[1]]); // Mark as yaml wildcard + } else if (showAll) { + results = yamlWildcards.map(x => [x[0].trim(), "yamlWildcard", x[1]]); // Mark as yaml wildcard + } + } else { + results = yamlWildcards.map(x => [x[0].trim(), "yamlWildcard", x[1]]); // Mark as yaml wildcard + } } else if (CFG.useEmbeddings && tagword.match(/<[^,> ]*>?/g)) { // Show embeddings let tempResults = []; @@ -662,13 +710,13 @@ async function autocomplete(textArea, prompt, fixedTag = null) { } // If onlyShowAlias is enabled, we don't need to include normal results if (CFG.alias.onlyShowAlias) { - results = allTags.filter(x => x[3] && x[3].toLowerCase().search(searchRegex) >- 1); + results = allTags.filter(x => x[3] && x[3].toLowerCase().search(searchRegex) > -1); } else { // Else both normal tags and aliases/translations are included depending on the config - let baseFilter = (x) => x[0].toLowerCase().search(searchRegex) >- 1; - let aliasFilter = (x) => x[3] && x[3].toLowerCase().search(searchRegex) >- 1; - let translationFilter = (x) => (translations.has(x[0]) && translations.get(x[0]).toLowerCase().search(searchRegex) >- 1) - || x[3] && x[3].split(",").some(y => translations.has(y) && translations.get(y).toLowerCase().search(searchRegex) >- 1); + let baseFilter = (x) => x[0].toLowerCase().search(searchRegex) > -1; + let aliasFilter = (x) => x[3] && x[3].toLowerCase().search(searchRegex) > -1; + let translationFilter = (x) => (translations.has(x[0]) && translations.get(x[0]).toLowerCase().search(searchRegex) > -1) + || x[3] && x[3].split(",").some(y => translations.has(y) && translations.get(y).toLowerCase().search(searchRegex) > -1); let fil; if (CFG.alias.searchByAlias && CFG.translation.searchByTranslation) @@ -822,6 +870,16 @@ async function setup() { console.error("Error loading wildcards: " + e); } } + // Load yaml wildcards + if (yamlWildcards.length === 0) { + try { + let yamlTags = (await readFile(`${tagBasePath}/temp/wcet.txt?${new Date().getTime()}`)).split("\n"); + // Split into tag, count pairs + yamlWildcards = yamlTags.map(x => x.trim().split(",")); + } catch (e) { + console.error("Error loading yaml wildcards: " + e); + } + } // Load embeddings if (embeddings.length === 0) { try { From 34ba08d80478db5b5d5c64ee19a0b50b6d3ccef3 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Tue, 20 Dec 2022 18:25:37 +0100 Subject: [PATCH 3/5] Extract regexes for easier editing & testing --- javascript/tagAutocomplete.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 5a21201..397e0ae 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -343,6 +343,9 @@ function escapeHTML(unsafeText) { const WEIGHT_REGEX = /[([]([^,()[\]:| ]+)(?::(?:\d+(?:\.\d+)?|\.\d+))?[)\]]/g; const TAG_REGEX = /(<[^\t\n\r,>]+>?|[^\s,|<>]+|<)/g +const WC_REGEX = /\b__([^, ]+)__([^, ]*)\b/g; +const UMI_PROMPT_REGEX = /<[^\s]*?\[[^,<>]*[\]|]?>?/g; +const UMI_TAG_REGEX = /(?:\[|\||--)([^<>\[\]\-|]+)/g; let hideBlocked = false; // On click, insert the tag into the prompt textbox with respect to the cursor position @@ -413,8 +416,8 @@ function insertTextAtCursor(textArea, result, tagword) { // If it was a yaml wildcard, also update the umiPreviousTags if (tagType === "yamlWildcard" && originalTagword.length > 0) { - let umiSubPrompt = originalTagword.match(/<[^\s]*?\[[^,<>]*[\]|]?>?/g)[0]; - umiPreviousTags = [...umiSubPrompt.matchAll(/(?:\[|\||--)([^<>\[\]\-|]+)/g)].map(x => x[1]); + let umiSubPrompt = originalTagword.match(UMI_PROMPT_REGEX)[0]; + umiPreviousTags = [...umiSubPrompt.matchAll(UMI_TAG_REGEX)].map(x => x[1]); } // Hide results after inserting @@ -616,9 +619,9 @@ async function autocomplete(textArea, prompt, fixedTag = null) { tagword = tagword.toLowerCase().replace(/[\n\r]/g, ""); - if (CFG.useWildcards && [...tagword.matchAll(/\b__([^, ]+)__([^, ]*)\b/g)].length > 0) { + if (CFG.useWildcards && [...tagword.matchAll(WC_REGEX)].length > 0) { // Show wildcards from a file with that name - wcMatch = [...tagword.matchAll(/\b__([^, ]+)__([^, ]*)\b/g)] + wcMatch = [...tagword.matchAll(WC_REGEX)] let wcFile = wcMatch[0][1]; let wcWord = wcMatch[0][2]; @@ -645,10 +648,10 @@ async function autocomplete(textArea, prompt, fixedTag = null) { tempResults = wildcardFiles.concat(wildcardExtFiles); } results = tempResults.map(x => ["Wildcards: " + x[1].trim(), "wildcardFile"]); // Mark as wildcard - } else if (CFG.useWildcards && tagword.match(/<[^\s]*?\[[^,<>]*[\]|]?>?/g)) { + } else if (CFG.useWildcards && tagword.match(UMI_PROMPT_REGEX)) { // We are in a UMI yaml tag definition, parse further - let umiSubPrompt = tagword.match(/<[^\s]*?\[[^,<>]*[\]|]?>?/g)[0]; - let umiTags = [...umiSubPrompt.matchAll(/(?:\[|\||--)([^<>\[\]\-|]+)/g)].map(x => x[1]); + let umiSubPrompt = tagword.match(UMI_PROMPT_REGEX)[0]; + let umiTags = [...umiSubPrompt.matchAll(UMI_TAG_REGEX)].map(x => x[1]); if (umiTags.length > 0) { // Get difference for subprompt let diff = difference(umiTags, umiPreviousTags); From 63a0d2e73ebdfb84d594dc04a7897b2dedbfe3bc Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Tue, 20 Dec 2022 19:45:07 +0100 Subject: [PATCH 4/5] Bugfixes for change detection & multiple umi tags (WIP) --- javascript/tagAutocomplete.js | 48 ++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 397e0ae..3c7389a 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -344,8 +344,8 @@ function escapeHTML(unsafeText) { const WEIGHT_REGEX = /[([]([^,()[\]:| ]+)(?::(?:\d+(?:\.\d+)?|\.\d+))?[)\]]/g; const TAG_REGEX = /(<[^\t\n\r,>]+>?|[^\s,|<>]+|<)/g const WC_REGEX = /\b__([^, ]+)__([^, ]*)\b/g; -const UMI_PROMPT_REGEX = /<[^\s]*?\[[^,<>]*[\]|]?>?/g; -const UMI_TAG_REGEX = /(?:\[|\||--)([^<>\[\]\-|]+)/g; +const UMI_PROMPT_REGEX = /<[^\s]*?\[[^,<>]*[\]|]?>?/gi; +const UMI_TAG_REGEX = /(?:\[|\||--)([^<>\[\]\-|]+)/gi; let hideBlocked = false; // On click, insert the tag into the prompt textbox with respect to the cursor position @@ -416,8 +416,24 @@ function insertTextAtCursor(textArea, result, tagword) { // If it was a yaml wildcard, also update the umiPreviousTags if (tagType === "yamlWildcard" && originalTagword.length > 0) { - let umiSubPrompt = originalTagword.match(UMI_PROMPT_REGEX)[0]; - umiPreviousTags = [...umiSubPrompt.matchAll(UMI_TAG_REGEX)].map(x => x[1]); + let editStart = Math.max(cursorPos - tagword.length, 0); + let editEnd = Math.min(cursorPos + tagword.length, originalTagword.length); + let surrounding = originalTagword.substring(editStart, editEnd); + let match = surrounding.match(new RegExp(escapeRegExp(`${tagword}`), "i")); + let insert = surrounding.replace(match, sanitizedText); + + let modifiedTagword = prompt.substring(0, editStart) + insert + prompt.substring(editEnd); + let umiSubPrompts = [...newPrompt.matchAll(UMI_PROMPT_REGEX)]; + + let umiTags = []; + umiSubPrompts.forEach(umiSubPrompt => { + umiTags = umiTags.concat([...umiSubPrompt[0].matchAll(UMI_TAG_REGEX)].map(x => x[1].toLowerCase())); + }); + + umiPreviousTags = umiTags; + + console.log("updated: " + umiPreviousTags) + hideResults(textArea); } // Hide results after inserting @@ -648,20 +664,29 @@ async function autocomplete(textArea, prompt, fixedTag = null) { tempResults = wildcardFiles.concat(wildcardExtFiles); } results = tempResults.map(x => ["Wildcards: " + x[1].trim(), "wildcardFile"]); // Mark as wildcard - } else if (CFG.useWildcards && tagword.match(UMI_PROMPT_REGEX)) { + } else if (CFG.useWildcards && [...tagword.matchAll(UMI_PROMPT_REGEX)].length > 0) { // We are in a UMI yaml tag definition, parse further - let umiSubPrompt = tagword.match(UMI_PROMPT_REGEX)[0]; - let umiTags = [...umiSubPrompt.matchAll(UMI_TAG_REGEX)].map(x => x[1]); + let umiSubPrompts = [...prompt.matchAll(UMI_PROMPT_REGEX)]; + + let umiTags = []; + umiSubPrompts.forEach(umiSubPrompt => { + umiTags = umiTags.concat([...umiSubPrompt[0].matchAll(UMI_TAG_REGEX)].map(x => x[1].toLowerCase())); + }); + if (umiTags.length > 0) { // Get difference for subprompt + let tagCountChange = umiTags.length - umiPreviousTags.length; let diff = difference(umiTags, umiPreviousTags); umiPreviousTags = umiTags; // Show all condition - let showAll = umiSubPrompt.endsWith("[") || umiSubPrompt.endsWith("[--") || umiSubPrompt.endsWith("|"); + let currentSubPrompt = umiSubPrompts.find(x => x[0].includes(tagword[0]))[0]; + let showAll = currentSubPrompt.endsWith("[") || currentSubPrompt.endsWith("[--") || currentSubPrompt.endsWith("|"); + + console.log(currentSubPrompt, umiTags, diff, tagCountChange) // Exit early if the user closed the bracket manually - if ((!diff || diff.length === 0) && (!showAll || umiSubPrompt.endsWith("]"))) { + if ((!diff || diff.length === 0 || (diff.length === 1 && tagCountChange < 0)) && !showAll) { if (!hideBlocked) hideResults(textArea); return; } @@ -669,6 +694,7 @@ async function autocomplete(textArea, prompt, fixedTag = null) { let umiTagword = diff[0]; let tempResults = []; if (umiTagword && umiTagword.length > 0) { + umiTagword = umiTagword.toLowerCase().replace(/[\n\r]/g, ""); originalTagword = tagword; tagword = umiTagword; @@ -679,9 +705,13 @@ async function autocomplete(textArea, prompt, fixedTag = null) { results = tempResults.map(x => [x[0].trim(), "yamlWildcard", x[1]]); // Mark as yaml wildcard } else if (showAll) { results = yamlWildcards.map(x => [x[0].trim(), "yamlWildcard", x[1]]); // Mark as yaml wildcard + originalTagword = tagword; + tagword = ""; } } else { results = yamlWildcards.map(x => [x[0].trim(), "yamlWildcard", x[1]]); // Mark as yaml wildcard + originalTagword = tagword; + tagword = ""; } } else if (CFG.useEmbeddings && tagword.match(/<[^,> ]*>?/g)) { // Show embeddings From bdd8cf68c7093294c5dff94cd98670fc6059e791 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Thu, 22 Dec 2022 12:39:00 +0100 Subject: [PATCH 5/5] Fix show all condition --- javascript/tagAutocomplete.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 3c7389a..29328dd 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -680,10 +680,9 @@ async function autocomplete(textArea, prompt, fixedTag = null) { umiPreviousTags = umiTags; // Show all condition - let currentSubPrompt = umiSubPrompts.find(x => x[0].includes(tagword[0]))[0]; - let showAll = currentSubPrompt.endsWith("[") || currentSubPrompt.endsWith("[--") || currentSubPrompt.endsWith("|"); + let showAll = tagword.endsWith("[") || tagword.endsWith("[--") || tagword.endsWith("|"); - console.log(currentSubPrompt, umiTags, diff, tagCountChange) + console.log(tagword, umiTags, diff, tagCountChange) // Exit early if the user closed the bracket manually if ((!diff || diff.length === 0 || (diff.length === 1 && tagCountChange < 0)) && !showAll) {