From e00199cf061d59356cf23c20d71b8e2cf12be7d8 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sat, 28 Jan 2023 22:36:16 +0100 Subject: [PATCH 01/26] Move potentially shared vars to separate file to ensure that they exist when other parts reference them --- javascript/_globals.js | 24 ++++++++++++++++++++++++ javascript/tagAutocomplete.js | 24 ------------------------ 2 files changed, 24 insertions(+), 24 deletions(-) create mode 100644 javascript/_globals.js diff --git a/javascript/_globals.js b/javascript/_globals.js new file mode 100644 index 0000000..c49757e --- /dev/null +++ b/javascript/_globals.js @@ -0,0 +1,24 @@ +var CFG = null; + +var tagBasePath = ""; +var allTags = []; +var translations = new Map(); + +var currentModelHash = ""; +var currentModelName = ""; + +var wildcardFiles = []; +var wildcardExtFiles = []; +var yamlWildcards = []; +var umiPreviousTags = []; +var embeddings = []; +var hypernetworks = []; +var loras = []; +var results = []; +var tagword = ""; +var originalTagword = ""; +var resultCount = 0; + +var selectedTag = null; +var oldSelectedTag = null; +var previousTags = []; \ No newline at end of file diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 062de4c..7473884 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -1,5 +1,3 @@ -var CFG = null; - const styleColors = { "--results-bg": ["#0b0f19", "#ffffff"], "--results-border-color": ["#4b5563", "#e5e7eb"], @@ -85,10 +83,6 @@ const autocompleteCSS = ` } `; -var tagBasePath = ""; -var allTags = []; -var translations = new Map(); - async function loadTags(c) { // Load main tags and aliases if (allTags.length === 0 && c.tagFile && c.tagFile !== "None") { @@ -248,10 +242,6 @@ function createResultsDiv(textArea) { return resultsDiv; } -// The selected tag index. Needs to be up here so hide can access it. -var selectedTag = null; -var previousTags = []; - // Show or hide the results div function isVisible(textArea) { let textAreaId = getTextAreaIdentifier(textArea); @@ -270,8 +260,6 @@ function hideResults(textArea) { selectedTag = null; } -var currentModelHash = ""; -var currentModelName = ""; // Function to check activation criteria function isEnabled() { if (CFG.activeIn.global) { @@ -569,17 +557,6 @@ function updateSelectionStyle(textArea, newIndex, oldIndex) { } } -var wildcardFiles = []; -var wildcardExtFiles = []; -var yamlWildcards = []; -var umiPreviousTags = []; -var embeddings = []; -var hypernetworks = []; -var loras = []; -var results = []; -var tagword = ""; -var originalTagword = ""; -var resultCount = 0; async function autocomplete(textArea, prompt, fixedTag = null) { // Return if the function is deactivated in the UI if (!isEnabled()) return; @@ -1023,7 +1000,6 @@ async function autocomplete(textArea, prompt, fixedTag = null) { addResultsToList(textArea, results, tagword, true); } -var oldSelectedTag = null; function navigateInList(textArea, event) { // Return if the function is deactivated in the UI or the current model is excluded due to white/blacklist settings if (!isEnabled()) return; From a831592c3cdfdf83527bd4288e852d75ce5db23b Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sat, 28 Jan 2023 22:57:58 +0100 Subject: [PATCH 02/26] Rename globals file to ensure it's loaded first --- javascript/__globals.js | 47 +++++++++++++++++++++++++++++++++++++++++ javascript/_globals.js | 24 --------------------- 2 files changed, 47 insertions(+), 24 deletions(-) create mode 100644 javascript/__globals.js delete mode 100644 javascript/_globals.js diff --git a/javascript/__globals.js b/javascript/__globals.js new file mode 100644 index 0000000..68e9493 --- /dev/null +++ b/javascript/__globals.js @@ -0,0 +1,47 @@ +// Core components +var CFG = null; +var tagBasePath = ""; + +// Tag completion data loaded from files +var allTags = []; +var translations = new Map(); +// Same for tag-likes +var wildcardFiles = []; +var wildcardExtFiles = []; +var yamlWildcards = []; +var embeddings = []; +var hypernetworks = []; +var loras = []; + +// Selected model info for black/whitelisting +var currentModelHash = ""; +var currentModelName = ""; + +// Current results +var results = []; +var resultCount = 0; + +// Relevant for parsing +var previousTags = []; +var tagword = ""; +var originalTagword = ""; + +// Tag selection for keyboard navigation +var selectedTag = null; +var oldSelectedTag = null; + +// UMI +var umiPreviousTags = []; + +/// Extendability system: +/// Provides "queues" for other files of the script (or really any js) +/// to add functions to be called at certain points in the script. +/// Similar to a callback system, but primitive. + +// Queues +var afterInsertQueue = []; +var afterSetupQueue = []; +var afterConfigChangeQueue = []; + +// List of parsers to try +var parsers = []; \ No newline at end of file diff --git a/javascript/_globals.js b/javascript/_globals.js deleted file mode 100644 index c49757e..0000000 --- a/javascript/_globals.js +++ /dev/null @@ -1,24 +0,0 @@ -var CFG = null; - -var tagBasePath = ""; -var allTags = []; -var translations = new Map(); - -var currentModelHash = ""; -var currentModelName = ""; - -var wildcardFiles = []; -var wildcardExtFiles = []; -var yamlWildcards = []; -var umiPreviousTags = []; -var embeddings = []; -var hypernetworks = []; -var loras = []; -var results = []; -var tagword = ""; -var originalTagword = ""; -var resultCount = 0; - -var selectedTag = null; -var oldSelectedTag = null; -var previousTags = []; \ No newline at end of file From b70b0b72cb030721d4722f200f8a4853ee9ae3b8 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sat, 28 Jan 2023 22:58:29 +0100 Subject: [PATCH 03/26] Add base parser --- javascript/_baseParser.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 javascript/_baseParser.js diff --git a/javascript/_baseParser.js b/javascript/_baseParser.js new file mode 100644 index 0000000..6c468cf --- /dev/null +++ b/javascript/_baseParser.js @@ -0,0 +1,14 @@ +class BaseTagParser { + triggerCondition = null; + + constructor (triggerCondition) { + if (new.target === BaseCompletionParser) { + throw new TypeError("Cannot construct abstract BaseCompletionParser directly"); + } + this.triggerCondition = triggerCondition; + } + + parse() { + throw new NotImplementedError(); + } +} \ No newline at end of file From d1d3cd2bf56b2e43a2f5c603d97125151485ecd3 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sat, 28 Jan 2023 23:28:15 +0100 Subject: [PATCH 04/26] Add queue processing & callbacks --- javascript/__globals.js | 8 ++++---- javascript/_utils.js | 19 +++++++++++++++++++ javascript/tagAutocomplete.js | 11 +++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/javascript/__globals.js b/javascript/__globals.js index 68e9493..cc918f2 100644 --- a/javascript/__globals.js +++ b/javascript/__globals.js @@ -39,9 +39,9 @@ var umiPreviousTags = []; /// Similar to a callback system, but primitive. // Queues -var afterInsertQueue = []; -var afterSetupQueue = []; -var afterConfigChangeQueue = []; +const afterInsertQueue = []; +const afterSetupQueue = []; +const afterConfigChangeQueue = []; // List of parsers to try -var parsers = []; \ No newline at end of file +const parsers = []; \ No newline at end of file diff --git a/javascript/_utils.js b/javascript/_utils.js index a659507..a36dbb3 100644 --- a/javascript/_utils.js +++ b/javascript/_utils.js @@ -93,4 +93,23 @@ function escapeHTML(unsafeText) { let div = document.createElement('div'); div.textContent = unsafeText; return div.innerHTML; +} + +// Queue calling function to process global queues +function processQueue(queue, context, ...args) { + for (let i = 0; i < queue.length; i++) { + queue[i].call(context, ...args); + } +} +// The same but with return values +function processQueueReturn(queue, context, ...args) +{ + let results = []; + for (let i = 0; i < queue.length; i++) { + results.push(queue[i].call(context, ...args)); + } + return results; +} +function processParsers(textArea, prompt) { + return processQueueReturn(parsers, null, textArea, prompt); } \ No newline at end of file diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 7473884..6f9e755 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -224,6 +224,9 @@ async function syncOptions() { // Apply changes CFG = newCFG; + + // Callback + processQueue(afterConfigChangeQueue, null); } // Create the result list div and necessary styling @@ -359,6 +362,9 @@ function insertTextAtCursor(textArea, result, tagword) { } previousTags = tags; + // Callback + processQueue(afterInsertQueue, null, tagType); + // If it was a yaml wildcard, also update the umiPreviousTags if (tagType === ResultType.yamlWildcard && originalTagword.length > 0) { let umiSubPrompts = [...newPrompt.matchAll(UMI_PROMPT_REGEX)]; @@ -612,6 +618,8 @@ async function autocomplete(textArea, prompt, fixedTag = null) { results = []; tagword = tagword.toLowerCase().replace(/[\n\r]/g, ""); + let resultCandidates = processParsers(textArea, prompt); + if (CFG.useWildcards && [...tagword.matchAll(WC_REGEX)].length > 0) { // Show wildcards from a file with that name wcMatch = [...tagword.matchAll(WC_REGEX)] @@ -1286,6 +1294,9 @@ async function setup() { acStyle.appendChild(document.createTextNode(css)); } gradioApp().appendChild(acStyle); + + // Callback + processQueue(afterSetupQueue, null); } onUiUpdate(async () => { From 98000bd2fcdb3cbd9945e0589c7195cbcd041cb8 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sat, 28 Jan 2023 23:46:22 +0100 Subject: [PATCH 05/26] Fix copy-paste error --- javascript/_baseParser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/_baseParser.js b/javascript/_baseParser.js index 6c468cf..b1ba284 100644 --- a/javascript/_baseParser.js +++ b/javascript/_baseParser.js @@ -2,7 +2,7 @@ class BaseTagParser { triggerCondition = null; constructor (triggerCondition) { - if (new.target === BaseCompletionParser) { + if (new.target === BaseTagParser) { throw new TypeError("Cannot construct abstract BaseCompletionParser directly"); } this.triggerCondition = triggerCondition; From 3e0a7cc79658469b95272f2b036e25f4bdbbd99f Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sun, 29 Jan 2023 00:37:39 +0100 Subject: [PATCH 06/26] Custom error for missing override --- javascript/_baseParser.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/javascript/_baseParser.js b/javascript/_baseParser.js index b1ba284..c88b8ef 100644 --- a/javascript/_baseParser.js +++ b/javascript/_baseParser.js @@ -1,3 +1,10 @@ +class FunctionNotOverriddenError extends Error { + constructor(message = "", ...args) { + super(message, ...args); + this.message = message + " is an abstract base function and must be overwritten."; + } +} + class BaseTagParser { triggerCondition = null; @@ -9,6 +16,6 @@ class BaseTagParser { } parse() { - throw new NotImplementedError(); + throw new FunctionNotOverriddenError("parse()"); } } \ No newline at end of file From cd807107083eaad1db53196219532eae05a7a29d Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sun, 29 Jan 2023 00:45:00 +0100 Subject: [PATCH 07/26] Implement parser queue --- javascript/_utils.js | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/javascript/_utils.js b/javascript/_utils.js index a36dbb3..08ba55d 100644 --- a/javascript/_utils.js +++ b/javascript/_utils.js @@ -102,14 +102,24 @@ function processQueue(queue, context, ...args) { } } // The same but with return values -function processQueueReturn(queue, context, ...args) +async function processQueueReturn(queue, context, ...args) { - let results = []; + let qeueueReturns = []; for (let i = 0; i < queue.length; i++) { - results.push(queue[i].call(context, ...args)); + qeueueReturns.push(await queue[i].call(context, ...args)); } - return results; + return qeueueReturns; } -function processParsers(textArea, prompt) { - return processQueueReturn(parsers, null, textArea, prompt); +// Specific to tag completion parsers +async function processParsers(textArea, prompt) { + // Get all parsers that have a successful trigger condition + let matchingParsers = parsers.filter(parser => parser.triggerCondition()); + // Guard condition + if (matchingParsers.length === 0) { + return null; + } + + let parseFunctions = matchingParsers.map(parser => parser.parse); + // Process them and return the results + return await processQueueReturn(parseFunctions, null, textArea, prompt); } \ No newline at end of file From 8e1422173942f29fc60e54d22e41dba31abd43a3 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sun, 29 Jan 2023 01:00:02 +0100 Subject: [PATCH 08/26] Extract wildcard completion --- javascript/ext_wildcards.js | 67 ++++ javascript/tagAutocomplete.js | 686 ++++++++++++++++------------------ 2 files changed, 395 insertions(+), 358 deletions(-) create mode 100644 javascript/ext_wildcards.js diff --git a/javascript/ext_wildcards.js b/javascript/ext_wildcards.js new file mode 100644 index 0000000..e2edf84 --- /dev/null +++ b/javascript/ext_wildcards.js @@ -0,0 +1,67 @@ +// Regex +const WC_REGEX = /\b__([^, ]+)__([^, ]*)\b/g; + +// Trigger conditions +const WC_TRIGGER = () => CFG.useWildcards && [...tagword.matchAll(WC_REGEX)].length > 0; +const WC_FILE_TRIGGER = () => CFG.useWildcards && (tagword.startsWith("__") && !tagword.endsWith("__") || tagword === "__"); + +class WildcardParser extends BaseTagParser { + constructor(TRIGGER) { + super(TRIGGER); + } + + async parse() { + // Show wildcards from a file with that name + let wcMatch = [...tagword.matchAll(WC_REGEX)] + let wcFile = wcMatch[0][1]; + let wcWord = wcMatch[0][2]; + + // Look in normal wildcard files + let wcFound = wildcardFiles.find(x => x[1].toLowerCase() === wcFile); + // Use found wildcard file or look in external wildcard files + let wcPair = wcFound || wildcardExtFiles.find(x => x[1].toLowerCase() === wcFile); + + let wildcards = (await readFile(`${wcPair[0]}/${wcPair[1]}.txt?${new Date().getTime()}`)).split("\n") + .filter(x => x.trim().length > 0 && !x.startsWith('#')); // Remove empty lines and comments + + let tempResults = wildcards.filter(x => (wcWord !== null && wcWord.length > 0) ? x.toLowerCase().includes(wcWord) : x) // Filter by tagword + .map(t => { + let result = new AutocompleteResult(t.trim(), ResultType.wildcardTag); + result.meta = wcFile; + return result; + }); + + return tempResults; + } +} + +class WildcardFileParser extends BaseTagParser { + constructor(TRIGGER) { + super(TRIGGER); + } + + parse() { + // Show available wildcard files + let tempResults = []; + if (tagword !== "__") { + let lmb = (x) => x[1].toLowerCase().includes(tagword.replace("__", "")) + tempResults = wildcardFiles.filter(lmb).concat(wildcardExtFiles.filter(lmb)) // Filter by tagword + } else { + tempResults = wildcardFiles.concat(wildcardExtFiles); + } + + let finalResults = []; + // Get final results + tempResults.forEach(wcFile => { + let result = new AutocompleteResult(wcFile[1].trim(), ResultType.wildcardFile); + result.meta = "Wildcard file"; + finalResults.push(result); + }); + + return finalResults; + } +} + +// Register the parsers +parsers.push(new WildcardParser(WC_TRIGGER)); +parsers.push(new WildcardFileParser(WC_FILE_TRIGGER)); \ No newline at end of file diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 6f9e755..7efd0cb 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -287,7 +287,6 @@ function isEnabled() { const WEIGHT_REGEX = /[([]([^,()[\]:| ]+)(?::(?:\d+(?:\.\d+)?|\.\d+))?[)\]]/g; const TAG_REGEX = /(<[^\t\n\r,>]+>?|[^\s,|<>]+|<)/g -const WC_REGEX = /\b__([^, ]+)__([^, ]*)\b/g; const UMI_PROMPT_REGEX = /<[^\s]*?\[[^,<>]*[\]|]?>?/gi; const UMI_TAG_REGEX = /(?:\[|\||--)([^<>\[\]\-|]+)/gi; let hideBlocked = false; @@ -618,382 +617,353 @@ async function autocomplete(textArea, prompt, fixedTag = null) { results = []; tagword = tagword.toLowerCase().replace(/[\n\r]/g, ""); - let resultCandidates = processParsers(textArea, prompt); - - if (CFG.useWildcards && [...tagword.matchAll(WC_REGEX)].length > 0) { - // Show wildcards from a file with that name - wcMatch = [...tagword.matchAll(WC_REGEX)] - let wcFile = wcMatch[0][1]; - let wcWord = wcMatch[0][2]; - - var wcPair; - - // Look in normal wildcard files - if (wcFound = wildcardFiles.find(x => x[1].toLowerCase() === wcFile)) - wcPair = wcFound; - else // Look in extensions wildcard files - wcPair = wildcardExtFiles.find(x => x[1].toLowerCase() === wcFile); - - let wildcards = (await readFile(`${wcPair[0]}/${wcPair[1]}.txt?${new Date().getTime()}`)).split("\n") - .filter(x => x.trim().length > 0 && !x.startsWith('#')); // Remove empty lines and comments - - - let tempResults = wildcards.filter(x => (wcWord !== null && wcWord.length > 0) ? x.toLowerCase().includes(wcWord) : x) // Filter by tagword - tempResults.forEach(t => { - let result = new AutocompleteResult(t.trim(), ResultType.wildcardTag); - result.meta = wcFile; - results.push(result); - }); - } else if (CFG.useWildcards && (tagword.startsWith("__") && !tagword.endsWith("__") || tagword === "__")) { - // Show available wildcard files - let tempResults = []; - if (tagword !== "__") { - let lmb = (x) => x[1].toLowerCase().includes(tagword.replace("__", "")) - tempResults = wildcardFiles.filter(lmb).concat(wildcardExtFiles.filter(lmb)) // Filter by tagword - } else { - tempResults = wildcardFiles.concat(wildcardExtFiles); + // Process all parsers + let resultCandidates = await processParsers(textArea, prompt); + if (resultCandidates && resultCandidates.length > 0) { + // Flatten our candidate(s) + results = resultCandidates.flat(); + // If there was more than one candidate, sort the results by text to mix them + // instead of having them added in the order of the parsers + let shouldSort = resultCandidates.length > 1; + if (shouldSort) { + results = results.sort((a, b) => a.text.localeCompare(b.text)); } - - // Add final results - tempResults.forEach(wcFile => { - let result = new AutocompleteResult(wcFile[1].trim(), ResultType.wildcardFile); - result.meta = "Wildcard file"; - results.push(result); - }) - } else if (CFG.useWildcards && [...tagword.matchAll(UMI_PROMPT_REGEX)].length > 0) { - // We are in a UMI yaml tag definition, parse further - let umiSubPrompts = [...prompt.matchAll(UMI_PROMPT_REGEX)]; - - let umiTags = []; - let umiTagsWithOperators = [] - - const insertAt = (str,char,pos) => str.slice(0,pos) + char + str.slice(pos); - - umiSubPrompts.forEach(umiSubPrompt => { - umiTags = umiTags.concat([...umiSubPrompt[0].matchAll(UMI_TAG_REGEX)].map(x => x[1].toLowerCase())); + } else { + if (CFG.useWildcards && [...tagword.matchAll(UMI_PROMPT_REGEX)].length > 0) { + // We are in a UMI yaml tag definition, parse further + let umiSubPrompts = [...prompt.matchAll(UMI_PROMPT_REGEX)]; - const start = umiSubPrompt.index; - const end = umiSubPrompt.index + umiSubPrompt[0].length; - if (textArea.selectionStart >= start && textArea.selectionStart <= end) { - umiTagsWithOperators = insertAt(umiSubPrompt[0], '###', textArea.selectionStart - start); - } - }); + let umiTags = []; + let umiTagsWithOperators = [] - const promptSplitToTags = umiTagsWithOperators.replace(']###[', '][').split("]["); + const insertAt = (str,char,pos) => str.slice(0,pos) + char + str.slice(pos); - const clean = (str) => str - .replaceAll('>', '') - .replaceAll('<', '') - .replaceAll('[', '') - .replaceAll(']', '') - .trim(); - - const matches = promptSplitToTags.reduce((acc, curr) => { - isOptional = curr.includes("|"); - isNegative = curr.startsWith("--"); - let out; - if (isOptional) { - out = { - hasCursor: curr.includes("###"), - tags: clean(curr).split('|').map(x => ({ - hasCursor: x.includes("###"), - isNegative: x.startsWith("--"), - tag: clean(x).replaceAll("###", '').replaceAll("--", '') - })) - }; - acc.optional.push(out); - acc.all.push(...out.tags.map(x => x.tag)); - } else if (isNegative) { - out = { - hasCursor: curr.includes("###"), - tags: clean(curr).replaceAll("###", '').split('|'), - }; - out.tags = out.tags.map(x => x.startsWith("--") ? x.substring(2) : x); - acc.negative.push(out); - acc.all.push(...out.tags); - } else { - out = { - hasCursor: curr.includes("###"), - tags: clean(curr).replaceAll("###", '').split('|'), - }; - acc.positive.push(out); - acc.all.push(...out.tags); - } - return acc; - }, { positive: [], negative: [], optional: [], all: [] }); - - //console.log({ matches }) - - const filteredWildcards = (tagword) => { - const wildcards = yamlWildcards.filter(x => { - let tags = x[1]; - const matchesNeg = - matches.negative.length === 0 - || matches.negative.every(x => - x.hasCursor - || x.tags.every(t => !tags[t]) - ); - if (!matchesNeg) return false; - const matchesPos = - matches.positive.length === 0 - || matches.positive.every(x => - x.hasCursor - || x.tags.every(t => tags[t]) - ); - if (!matchesPos) return false; - const matchesOpt = - matches.optional.length === 0 - || matches.optional.some(x => - x.tags.some(t => - t.hasCursor - || t.isNegative - ? !tags[t.tag] - : tags[t.tag] - )); - if (!matchesOpt) return false; - return true; - }).reduce((acc, val) => { - Object.keys(val[1]).forEach(tag => acc[tag] = acc[tag] + 1 || 1); - return acc; - }, {}); - - return Object.entries(wildcards) - .sort((a, b) => b[1] - a[1]) - .filter(x => - x[0] === tagword - || !matches.all.includes(x[0]) - ); - } - - if (umiTags.length > 0) { - // Get difference for subprompt - let tagCountChange = umiTags.length - umiPreviousTags.length; - let diff = difference(umiTags, umiPreviousTags); - umiPreviousTags = umiTags; - - // Show all condition - let showAll = tagword.endsWith("[") || tagword.endsWith("[--") || tagword.endsWith("|"); - - // Exit early if the user closed the bracket manually - if ((!diff || diff.length === 0 || (diff.length === 1 && tagCountChange < 0)) && !showAll) { - if (!hideBlocked) hideResults(textArea); - return; - } - - let umiTagword = diff[0] || ''; - let tempResults = []; - if (umiTagword && umiTagword.length > 0) { - umiTagword = umiTagword.toLowerCase().replace(/[\n\r]/g, ""); - originalTagword = tagword; - tagword = umiTagword; - let filteredWildcardsSorted = filteredWildcards(umiTagword); - let searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(umiTagword)}`, 'i') - let baseFilter = x => x[0].toLowerCase().search(searchRegex) > -1; - let spaceIncludeFilter = x => x[0].toLowerCase().replaceAll(" ", "_").search(searchRegex) > -1; - tempResults = filteredWildcardsSorted.filter(x => baseFilter(x) || spaceIncludeFilter(x)) // Filter by tagword - - // Add final results - tempResults.forEach(t => { - let result = new AutocompleteResult(t[0].trim(), ResultType.yamlWildcard) - result.count = t[1]; - results.push(result); - }); - } else if (showAll) { - let filteredWildcardsSorted = filteredWildcards(""); + umiSubPrompts.forEach(umiSubPrompt => { + umiTags = umiTags.concat([...umiSubPrompt[0].matchAll(UMI_TAG_REGEX)].map(x => x[1].toLowerCase())); + const start = umiSubPrompt.index; + const end = umiSubPrompt.index + umiSubPrompt[0].length; + if (textArea.selectionStart >= start && textArea.selectionStart <= end) { + umiTagsWithOperators = insertAt(umiSubPrompt[0], '###', textArea.selectionStart - start); + } + }); + + const promptSplitToTags = umiTagsWithOperators.replace(']###[', '][').split("]["); + + const clean = (str) => str + .replaceAll('>', '') + .replaceAll('<', '') + .replaceAll('[', '') + .replaceAll(']', '') + .trim(); + + const matches = promptSplitToTags.reduce((acc, curr) => { + isOptional = curr.includes("|"); + isNegative = curr.startsWith("--"); + let out; + if (isOptional) { + out = { + hasCursor: curr.includes("###"), + tags: clean(curr).split('|').map(x => ({ + hasCursor: x.includes("###"), + isNegative: x.startsWith("--"), + tag: clean(x).replaceAll("###", '').replaceAll("--", '') + })) + }; + acc.optional.push(out); + acc.all.push(...out.tags.map(x => x.tag)); + } else if (isNegative) { + out = { + hasCursor: curr.includes("###"), + tags: clean(curr).replaceAll("###", '').split('|'), + }; + out.tags = out.tags.map(x => x.startsWith("--") ? x.substring(2) : x); + acc.negative.push(out); + acc.all.push(...out.tags); + } else { + out = { + hasCursor: curr.includes("###"), + tags: clean(curr).replaceAll("###", '').split('|'), + }; + acc.positive.push(out); + acc.all.push(...out.tags); + } + return acc; + }, { positive: [], negative: [], optional: [], all: [] }); + + //console.log({ matches }) + + const filteredWildcards = (tagword) => { + const wildcards = yamlWildcards.filter(x => { + let tags = x[1]; + const matchesNeg = + matches.negative.length === 0 + || matches.negative.every(x => + x.hasCursor + || x.tags.every(t => !tags[t]) + ); + if (!matchesNeg) return false; + const matchesPos = + matches.positive.length === 0 + || matches.positive.every(x => + x.hasCursor + || x.tags.every(t => tags[t]) + ); + if (!matchesPos) return false; + const matchesOpt = + matches.optional.length === 0 + || matches.optional.some(x => + x.tags.some(t => + t.hasCursor + || t.isNegative + ? !tags[t.tag] + : tags[t.tag] + )); + if (!matchesOpt) return false; + return true; + }).reduce((acc, val) => { + Object.keys(val[1]).forEach(tag => acc[tag] = acc[tag] + 1 || 1); + return acc; + }, {}); + + return Object.entries(wildcards) + .sort((a, b) => b[1] - a[1]) + .filter(x => + x[0] === tagword + || !matches.all.includes(x[0]) + ); + } + + if (umiTags.length > 0) { + // Get difference for subprompt + let tagCountChange = umiTags.length - umiPreviousTags.length; + let diff = difference(umiTags, umiPreviousTags); + umiPreviousTags = umiTags; + + // Show all condition + let showAll = tagword.endsWith("[") || tagword.endsWith("[--") || tagword.endsWith("|"); + + // Exit early if the user closed the bracket manually + if ((!diff || diff.length === 0 || (diff.length === 1 && tagCountChange < 0)) && !showAll) { + if (!hideBlocked) hideResults(textArea); + return; + } + + let umiTagword = diff[0] || ''; + let tempResults = []; + if (umiTagword && umiTagword.length > 0) { + umiTagword = umiTagword.toLowerCase().replace(/[\n\r]/g, ""); + originalTagword = tagword; + tagword = umiTagword; + let filteredWildcardsSorted = filteredWildcards(umiTagword); + let searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(umiTagword)}`, 'i') + let baseFilter = x => x[0].toLowerCase().search(searchRegex) > -1; + let spaceIncludeFilter = x => x[0].toLowerCase().replaceAll(" ", "_").search(searchRegex) > -1; + tempResults = filteredWildcardsSorted.filter(x => baseFilter(x) || spaceIncludeFilter(x)) // Filter by tagword + + // Add final results + tempResults.forEach(t => { + let result = new AutocompleteResult(t[0].trim(), ResultType.yamlWildcard) + result.count = t[1]; + results.push(result); + }); + } else if (showAll) { + let filteredWildcardsSorted = filteredWildcards(""); + + // Add final results + filteredWildcardsSorted.forEach(t => { + let result = new AutocompleteResult(t[0].trim(), ResultType.yamlWildcard) + result.count = t[1]; + results.push(result); + }); + + originalTagword = tagword; + tagword = ""; + } + } else { + let filteredWildcardsSorted = filteredWildcards(""); + // Add final results filteredWildcardsSorted.forEach(t => { let result = new AutocompleteResult(t[0].trim(), ResultType.yamlWildcard) result.count = t[1]; results.push(result); }); - + originalTagword = tagword; tagword = ""; } - } else { - let filteredWildcardsSorted = filteredWildcards(""); - - // Add final results - filteredWildcardsSorted.forEach(t => { - let result = new AutocompleteResult(t[0].trim(), ResultType.yamlWildcard) - result.count = t[1]; - results.push(result); - }); - - originalTagword = tagword; - tagword = ""; - } - } else if (CFG.useEmbeddings && tagword.match(/ ]*>?/g)) { - // Show embeddings - let tempResults = []; - if (tagword !== " x[0].toLowerCase().includes(searchTerm) && x[1] && x[1] === versionString); // Filter by tagword - else - tempResults = embeddings.filter(x => x[0].toLowerCase().includes(searchTerm)); // Filter by tagword - } else { - tempResults = embeddings; - } - - // Add final results - tempResults.forEach(t => { - let result = new AutocompleteResult(t[0].trim(), ResultType.embedding) - result.meta = t[1] + " Embedding"; - results.push(result); - }); - } else if(CFG.useHypernetworks && tagword.match(/ ]*>?/g)) { - // Show hypernetworks - let tempResults = []; - if (tagword !== " x.toLowerCase().includes(searchTerm)); // Filter by tagword - } else { - tempResults = hypernetworks; - } - - // Add final results - tempResults.forEach(t => { - let result = new AutocompleteResult(t.trim(), ResultType.hypernetwork) - result.meta = "Hypernetwork"; - results.push(result); - }); - } else if(CFG.useLoras && tagword.match(/ ]*>?/g)){ - // Show lora - let tempResults = []; - if (tagword !== " x.toLowerCase().includes(searchTerm)); // Filter by tagword - } else { - tempResults = loras; - } - - // Add final results - tempResults.forEach(t => { - let result = new AutocompleteResult(t.trim(), ResultType.lora) - result.meta = "Lora"; - results.push(result); - }); - } else if ((CFG.useEmbeddings || CFG.useHypernetworks || CFG.useLoras) && tagword.match(/<[^,> ]*>?/g)) { - // Embeddings, lora, wildcards all together with generic options - let tempEmbResults = []; - let tempHypResults = []; - let tempLoraResults = []; - if (tagword !== "<") { - let searchTerm = tagword.replace("<", "") - - let versionString; - if (searchTerm.startsWith("v1") || searchTerm.startsWith("v2")) { - versionString = searchTerm.slice(0, 2); - searchTerm = searchTerm.slice(2); - } - - if (versionString && CFG.useEmbeddings) { - // Version string is only for embeddings atm, so we don't search the other lists here. - tempEmbResults = embeddings.filter(x => x[0].toLowerCase().includes(searchTerm) && x[1] && x[1] === versionString); // Filter by tagword + } else if (CFG.useEmbeddings && tagword.match(/ ]*>?/g)) { + // Show embeddings + let tempResults = []; + if (tagword !== " x[0].toLowerCase().includes(searchTerm) && x[1] && x[1] === versionString); // Filter by tagword + else + tempResults = embeddings.filter(x => x[0].toLowerCase().includes(searchTerm)); // Filter by tagword } else { - tempEmbResults = embeddings.filter(x => x[0].toLowerCase().includes(searchTerm)); // Filter by tagword - tempHypResults = hypernetworks.filter(x => x.toLowerCase().includes(searchTerm)); // Filter by tagword - tempLoraResults = loras.filter(x => x.toLowerCase().includes(searchTerm)); // Filter by tagword + tempResults = embeddings; } - } else { - tempEmbResults = embeddings; - tempHypResults = hypernetworks; - tempLoraResults = loras; - } - // Since some tags are kaomoji, we have to still get the normal results first. - // Create escaped search regex with support for * as a start placeholder - let searchRegex; - if (tagword.startsWith("*")) { - tagword = tagword.slice(1); - searchRegex = new RegExp(`${escapeRegExp(tagword)}`, 'i'); - } else { - searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(tagword)}`, 'i'); - } - let genericResults = allTags.filter(x => x[0].toLowerCase().search(searchRegex) > -1).slice(0, CFG.maxResults); - - // Add final results - let mixedResults = []; - if (CFG.useEmbeddings) { - tempEmbResults.forEach(t => { + // Add final results + tempResults.forEach(t => { let result = new AutocompleteResult(t[0].trim(), ResultType.embedding) result.meta = t[1] + " Embedding"; - mixedResults.push(result); - }); - } - if (CFG.useHypernetworks) { - tempHypResults.forEach(t => { - let result = new AutocompleteResult(t.trim(), ResultType.hypernetwork) - result.meta = "Hypernetwork"; - mixedResults.push(result); - }); - } - if (CFG.useLoras) { - tempLoraResults.forEach(t => { - let result = new AutocompleteResult(t.trim(), ResultType.lora) - result.meta = "Lora"; - mixedResults.push(result); - }); - } - - // Add all mixed results to the final results, sorted by name so that they aren't after one another. - results = mixedResults.sort((a, b) => a.text.localeCompare(b.text)); - - genericResults.forEach(g => { - let result = new AutocompleteResult(g[0].trim(), ResultType.tag) - result.category = g[1]; - result.count = g[2]; - result.aliases = g[3]; - results.push(result); - }); - } else { - // Create escaped search regex with support for * as a start placeholder - let searchRegex; - if (tagword.startsWith("*")) { - tagword = tagword.slice(1); - searchRegex = new RegExp(`${escapeRegExp(tagword)}`, 'i'); - } else { - searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(tagword)}`, 'i'); - } - // If onlyShowAlias is enabled, we don't need to include normal results - if (CFG.alias.onlyShowAlias) { - results = allTags.filter(x => x[3] && x[3].toLowerCase().search(searchRegex) > -1); - } else { - // Else both normal tags and aliases/translations are included depending on the config - let baseFilter = (x) => x[0].toLowerCase().search(searchRegex) > -1; - let aliasFilter = (x) => x[3] && x[3].toLowerCase().search(searchRegex) > -1; - let translationFilter = (x) => (translations.has(x[0]) && translations.get(x[0]).toLowerCase().search(searchRegex) > -1) - || x[3] && x[3].split(",").some(y => translations.has(y) && translations.get(y).toLowerCase().search(searchRegex) > -1); - - let fil; - if (CFG.alias.searchByAlias && CFG.translation.searchByTranslation) - fil = (x) => baseFilter(x) || aliasFilter(x) || translationFilter(x); - else if (CFG.alias.searchByAlias && !CFG.translation.searchByTranslation) - fil = (x) => baseFilter(x) || aliasFilter(x); - else if (CFG.translation.searchByTranslation && !CFG.alias.searchByAlias) - fil = (x) => baseFilter(x) || translationFilter(x); - else - fil = (x) => baseFilter(x); - - // Add final results - allTags.filter(fil).forEach(t => { - let result = new AutocompleteResult(t[0].trim(), ResultType.tag) - result.category = t[1]; - result.count = t[2]; - result.aliases = t[3]; results.push(result); }); - } - // Slice if the user has set a max result count - if (!CFG.showAllResults) { - results = results.slice(0, CFG.maxResults); + } else if(CFG.useHypernetworks && tagword.match(/ ]*>?/g)) { + // Show hypernetworks + let tempResults = []; + if (tagword !== " x.toLowerCase().includes(searchTerm)); // Filter by tagword + } else { + tempResults = hypernetworks; + } + + // Add final results + tempResults.forEach(t => { + let result = new AutocompleteResult(t.trim(), ResultType.hypernetwork) + result.meta = "Hypernetwork"; + results.push(result); + }); + } else if(CFG.useLoras && tagword.match(/ ]*>?/g)){ + // Show lora + let tempResults = []; + if (tagword !== " x.toLowerCase().includes(searchTerm)); // Filter by tagword + } else { + tempResults = loras; + } + + // Add final results + tempResults.forEach(t => { + let result = new AutocompleteResult(t.trim(), ResultType.lora) + result.meta = "Lora"; + results.push(result); + }); + } else if ((CFG.useEmbeddings || CFG.useHypernetworks || CFG.useLoras) && tagword.match(/<[^,> ]*>?/g)) { + // Embeddings, lora, wildcards all together with generic options + let tempEmbResults = []; + let tempHypResults = []; + let tempLoraResults = []; + if (tagword !== "<") { + let searchTerm = tagword.replace("<", "") + + let versionString; + if (searchTerm.startsWith("v1") || searchTerm.startsWith("v2")) { + versionString = searchTerm.slice(0, 2); + searchTerm = searchTerm.slice(2); + } + + if (versionString && CFG.useEmbeddings) { + // Version string is only for embeddings atm, so we don't search the other lists here. + tempEmbResults = embeddings.filter(x => x[0].toLowerCase().includes(searchTerm) && x[1] && x[1] === versionString); // Filter by tagword + } else { + tempEmbResults = embeddings.filter(x => x[0].toLowerCase().includes(searchTerm)); // Filter by tagword + tempHypResults = hypernetworks.filter(x => x.toLowerCase().includes(searchTerm)); // Filter by tagword + tempLoraResults = loras.filter(x => x.toLowerCase().includes(searchTerm)); // Filter by tagword + } + } else { + tempEmbResults = embeddings; + tempHypResults = hypernetworks; + tempLoraResults = loras; + } + + // Since some tags are kaomoji, we have to still get the normal results first. + // Create escaped search regex with support for * as a start placeholder + let searchRegex; + if (tagword.startsWith("*")) { + tagword = tagword.slice(1); + searchRegex = new RegExp(`${escapeRegExp(tagword)}`, 'i'); + } else { + searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(tagword)}`, 'i'); + } + let genericResults = allTags.filter(x => x[0].toLowerCase().search(searchRegex) > -1).slice(0, CFG.maxResults); + + // Add final results + let mixedResults = []; + if (CFG.useEmbeddings) { + tempEmbResults.forEach(t => { + let result = new AutocompleteResult(t[0].trim(), ResultType.embedding) + result.meta = t[1] + " Embedding"; + mixedResults.push(result); + }); + } + if (CFG.useHypernetworks) { + tempHypResults.forEach(t => { + let result = new AutocompleteResult(t.trim(), ResultType.hypernetwork) + result.meta = "Hypernetwork"; + mixedResults.push(result); + }); + } + if (CFG.useLoras) { + tempLoraResults.forEach(t => { + let result = new AutocompleteResult(t.trim(), ResultType.lora) + result.meta = "Lora"; + mixedResults.push(result); + }); + } + + // Add all mixed results to the final results, sorted by name so that they aren't after one another. + results = mixedResults.sort((a, b) => a.text.localeCompare(b.text)); + + genericResults.forEach(g => { + let result = new AutocompleteResult(g[0].trim(), ResultType.tag) + result.category = g[1]; + result.count = g[2]; + result.aliases = g[3]; + results.push(result); + }); + } else { + // Create escaped search regex with support for * as a start placeholder + let searchRegex; + if (tagword.startsWith("*")) { + tagword = tagword.slice(1); + searchRegex = new RegExp(`${escapeRegExp(tagword)}`, 'i'); + } else { + searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(tagword)}`, 'i'); + } + // If onlyShowAlias is enabled, we don't need to include normal results + if (CFG.alias.onlyShowAlias) { + results = allTags.filter(x => x[3] && x[3].toLowerCase().search(searchRegex) > -1); + } else { + // Else both normal tags and aliases/translations are included depending on the config + let baseFilter = (x) => x[0].toLowerCase().search(searchRegex) > -1; + let aliasFilter = (x) => x[3] && x[3].toLowerCase().search(searchRegex) > -1; + let translationFilter = (x) => (translations.has(x[0]) && translations.get(x[0]).toLowerCase().search(searchRegex) > -1) + || x[3] && x[3].split(",").some(y => translations.has(y) && translations.get(y).toLowerCase().search(searchRegex) > -1); + + let fil; + if (CFG.alias.searchByAlias && CFG.translation.searchByTranslation) + fil = (x) => baseFilter(x) || aliasFilter(x) || translationFilter(x); + else if (CFG.alias.searchByAlias && !CFG.translation.searchByTranslation) + fil = (x) => baseFilter(x) || aliasFilter(x); + else if (CFG.translation.searchByTranslation && !CFG.alias.searchByAlias) + fil = (x) => baseFilter(x) || translationFilter(x); + else + fil = (x) => baseFilter(x); + + // Add final results + allTags.filter(fil).forEach(t => { + let result = new AutocompleteResult(t[0].trim(), ResultType.tag) + result.category = t[1]; + result.count = t[2]; + result.aliases = t[3]; + results.push(result); + }); + } + // Slice if the user has set a max result count + if (!CFG.showAllResults) { + results = results.slice(0, CFG.maxResults); + } } } @@ -1004,8 +974,8 @@ async function autocomplete(textArea, prompt, fixedTag = null) { return; } - showResults(textArea); addResultsToList(textArea, results, tagword, true); + showResults(textArea); } function navigateInList(textArea, event) { @@ -1096,7 +1066,7 @@ async function setup() { CFG["colors"] = (await readFile(`${tagBasePath}/colors.json?${new Date().getTime()}`, true)); // Load wildcards - if (wildcardFiles.length === 0) { + if (wildcardFiles.length === 0 && wildcardExtFiles.length === 0) { try { let wcFileArr = (await readFile(`${tagBasePath}/temp/wc.txt?${new Date().getTime()}`)).split("\n"); let wcBasePath = wcFileArr[0].trim(); // First line should be the base path From 29d1e7212d06b805ed898c57b3ffae5411c6eea6 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sun, 29 Jan 2023 16:40:04 +0100 Subject: [PATCH 09/26] Rename queues to fit const naming convention --- javascript/__globals.js | 8 ++++---- javascript/_utils.js | 2 +- javascript/tagAutocomplete.js | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/javascript/__globals.js b/javascript/__globals.js index cc918f2..dacc065 100644 --- a/javascript/__globals.js +++ b/javascript/__globals.js @@ -39,9 +39,9 @@ var umiPreviousTags = []; /// Similar to a callback system, but primitive. // Queues -const afterInsertQueue = []; -const afterSetupQueue = []; -const afterConfigChangeQueue = []; +const QUEUE_AFTER_INSERT = []; +const QUEUE_AFTER_SETUP = []; +const QUEUE_AFTER_CONFIG_CHANGE = []; // List of parsers to try -const parsers = []; \ No newline at end of file +const PARSERS = []; \ No newline at end of file diff --git a/javascript/_utils.js b/javascript/_utils.js index 08ba55d..be54860 100644 --- a/javascript/_utils.js +++ b/javascript/_utils.js @@ -113,7 +113,7 @@ async function processQueueReturn(queue, context, ...args) // Specific to tag completion parsers async function processParsers(textArea, prompt) { // Get all parsers that have a successful trigger condition - let matchingParsers = parsers.filter(parser => parser.triggerCondition()); + let matchingParsers = PARSERS.filter(parser => parser.triggerCondition()); // Guard condition if (matchingParsers.length === 0) { return null; diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 7efd0cb..22c98e8 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -226,7 +226,7 @@ async function syncOptions() { CFG = newCFG; // Callback - processQueue(afterConfigChangeQueue, null); + processQueue(QUEUE_AFTER_CONFIG_CHANGE, null); } // Create the result list div and necessary styling @@ -362,7 +362,7 @@ function insertTextAtCursor(textArea, result, tagword) { previousTags = tags; // Callback - processQueue(afterInsertQueue, null, tagType); + processQueue(QUEUE_AFTER_INSERT, null, tagType); // If it was a yaml wildcard, also update the umiPreviousTags if (tagType === ResultType.yamlWildcard && originalTagword.length > 0) { @@ -1266,7 +1266,7 @@ async function setup() { gradioApp().appendChild(acStyle); // Callback - processQueue(afterSetupQueue, null); + processQueue(QUEUE_AFTER_SETUP, null); } onUiUpdate(async () => { From 86fafeebf5a3f79e01a75d4750624ea7a47fc3e6 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sun, 29 Jan 2023 17:14:05 +0100 Subject: [PATCH 10/26] Fix for undefined returns --- javascript/_utils.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/javascript/_utils.js b/javascript/_utils.js index be54860..a8251b9 100644 --- a/javascript/_utils.js +++ b/javascript/_utils.js @@ -106,7 +106,9 @@ async function processQueueReturn(queue, context, ...args) { let qeueueReturns = []; for (let i = 0; i < queue.length; i++) { - qeueueReturns.push(await queue[i].call(context, ...args)); + let returnValue = await queue[i].call(context, ...args); + if (returnValue) + qeueueReturns.push(returnValue); } return qeueueReturns; } From 93ee32175d231a8283fc9262dbe5420a731d9fe4 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sun, 29 Jan 2023 17:16:09 +0100 Subject: [PATCH 11/26] Wildcard fixes & cleanup --- javascript/ext_wildcards.js | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/javascript/ext_wildcards.js b/javascript/ext_wildcards.js index e2edf84..18143ef 100644 --- a/javascript/ext_wildcards.js +++ b/javascript/ext_wildcards.js @@ -6,10 +6,6 @@ const WC_TRIGGER = () => CFG.useWildcards && [...tagword.matchAll(WC_REGEX)].len const WC_FILE_TRIGGER = () => CFG.useWildcards && (tagword.startsWith("__") && !tagword.endsWith("__") || tagword === "__"); class WildcardParser extends BaseTagParser { - constructor(TRIGGER) { - super(TRIGGER); - } - async parse() { // Show wildcards from a file with that name let wcMatch = [...tagword.matchAll(WC_REGEX)] @@ -24,22 +20,19 @@ class WildcardParser extends BaseTagParser { let wildcards = (await readFile(`${wcPair[0]}/${wcPair[1]}.txt?${new Date().getTime()}`)).split("\n") .filter(x => x.trim().length > 0 && !x.startsWith('#')); // Remove empty lines and comments + let finalResults = []; let tempResults = wildcards.filter(x => (wcWord !== null && wcWord.length > 0) ? x.toLowerCase().includes(wcWord) : x) // Filter by tagword - .map(t => { - let result = new AutocompleteResult(t.trim(), ResultType.wildcardTag); - result.meta = wcFile; - return result; - }); + tempResults.forEach(t => { + let result = new AutocompleteResult(t.trim(), ResultType.wildcardTag); + result.meta = wcFile; + finalResults.push(result); + }); - return tempResults; + return finalResults; } } class WildcardFileParser extends BaseTagParser { - constructor(TRIGGER) { - super(TRIGGER); - } - parse() { // Show available wildcard files let tempResults = []; @@ -63,5 +56,5 @@ class WildcardFileParser extends BaseTagParser { } // Register the parsers -parsers.push(new WildcardParser(WC_TRIGGER)); -parsers.push(new WildcardFileParser(WC_FILE_TRIGGER)); \ No newline at end of file +PARSERS.push(new WildcardParser(WC_TRIGGER)); +PARSERS.push(new WildcardFileParser(WC_FILE_TRIGGER)); \ No newline at end of file From 95eb9dd6e9e522cb575947d9a71dadd9b7fc0098 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sun, 29 Jan 2023 17:19:15 +0100 Subject: [PATCH 12/26] Extract UMI completion (base) --- javascript/ext_umi.js | 181 ++++++++++++++++++++++++++++++++++ javascript/tagAutocomplete.js | 169 +------------------------------ 2 files changed, 183 insertions(+), 167 deletions(-) create mode 100644 javascript/ext_umi.js diff --git a/javascript/ext_umi.js b/javascript/ext_umi.js new file mode 100644 index 0000000..0fe7fad --- /dev/null +++ b/javascript/ext_umi.js @@ -0,0 +1,181 @@ +const UMI_PROMPT_REGEX = /<[^\s]*?\[[^,<>]*[\]|]?>?/gi; +const UMI_TAG_REGEX = /(?:\[|\||--)([^<>\[\]\-|]+)/gi; + +const UMI_TRIGGER = () => CFG.useWildcards && [...tagword.matchAll(UMI_PROMPT_REGEX)].length > 0; + +class UmiParser extends BaseTagParser { + parse(textArea, prompt) { + // We are in a UMI yaml tag definition, parse further + let umiSubPrompts = [...prompt.matchAll(UMI_PROMPT_REGEX)]; + + let umiTags = []; + let umiTagsWithOperators = [] + + const insertAt = (str,char,pos) => str.slice(0,pos) + char + str.slice(pos); + + umiSubPrompts.forEach(umiSubPrompt => { + umiTags = umiTags.concat([...umiSubPrompt[0].matchAll(UMI_TAG_REGEX)].map(x => x[1].toLowerCase())); + + const start = umiSubPrompt.index; + const end = umiSubPrompt.index + umiSubPrompt[0].length; + if (textArea.selectionStart >= start && textArea.selectionStart <= end) { + umiTagsWithOperators = insertAt(umiSubPrompt[0], '###', textArea.selectionStart - start); + } + }); + + const promptSplitToTags = umiTagsWithOperators.replace(']###[', '][').split("]["); + + const clean = (str) => str + .replaceAll('>', '') + .replaceAll('<', '') + .replaceAll('[', '') + .replaceAll(']', '') + .trim(); + + const matches = promptSplitToTags.reduce((acc, curr) => { + let isOptional = curr.includes("|"); + let isNegative = curr.startsWith("--"); + let out; + if (isOptional) { + out = { + hasCursor: curr.includes("###"), + tags: clean(curr).split('|').map(x => ({ + hasCursor: x.includes("###"), + isNegative: x.startsWith("--"), + tag: clean(x).replaceAll("###", '').replaceAll("--", '') + })) + }; + acc.optional.push(out); + acc.all.push(...out.tags.map(x => x.tag)); + } else if (isNegative) { + out = { + hasCursor: curr.includes("###"), + tags: clean(curr).replaceAll("###", '').split('|'), + }; + out.tags = out.tags.map(x => x.startsWith("--") ? x.substring(2) : x); + acc.negative.push(out); + acc.all.push(...out.tags); + } else { + out = { + hasCursor: curr.includes("###"), + tags: clean(curr).replaceAll("###", '').split('|'), + }; + acc.positive.push(out); + acc.all.push(...out.tags); + } + return acc; + }, { positive: [], negative: [], optional: [], all: [] }); + + //console.log({ matches }) + + const filteredWildcards = (tagword) => { + const wildcards = yamlWildcards.filter(x => { + let tags = x[1]; + const matchesNeg = + matches.negative.length === 0 + || matches.negative.every(x => + x.hasCursor + || x.tags.every(t => !tags[t]) + ); + if (!matchesNeg) return false; + const matchesPos = + matches.positive.length === 0 + || matches.positive.every(x => + x.hasCursor + || x.tags.every(t => tags[t]) + ); + if (!matchesPos) return false; + const matchesOpt = + matches.optional.length === 0 + || matches.optional.some(x => + x.tags.some(t => + t.hasCursor + || t.isNegative + ? !tags[t.tag] + : tags[t.tag] + )); + if (!matchesOpt) return false; + return true; + }).reduce((acc, val) => { + Object.keys(val[1]).forEach(tag => acc[tag] = acc[tag] + 1 || 1); + return acc; + }, {}); + + return Object.entries(wildcards) + .sort((a, b) => b[1] - a[1]) + .filter(x => + x[0] === tagword + || !matches.all.includes(x[0]) + ); + } + + if (umiTags.length > 0) { + // Get difference for subprompt + let tagCountChange = umiTags.length - umiPreviousTags.length; + let diff = difference(umiTags, umiPreviousTags); + umiPreviousTags = umiTags; + + // Show all condition + let showAll = tagword.endsWith("[") || tagword.endsWith("[--") || tagword.endsWith("|"); + + // Exit early if the user closed the bracket manually + if ((!diff || diff.length === 0 || (diff.length === 1 && tagCountChange < 0)) && !showAll) { + if (!hideBlocked) hideResults(textArea); + return; + } + + let umiTagword = diff[0] || ''; + let tempResults = []; + if (umiTagword && umiTagword.length > 0) { + umiTagword = umiTagword.toLowerCase().replace(/[\n\r]/g, ""); + originalTagword = tagword; + tagword = umiTagword; + let filteredWildcardsSorted = filteredWildcards(umiTagword); + let searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(umiTagword)}`, 'i') + let baseFilter = x => x[0].toLowerCase().search(searchRegex) > -1; + let spaceIncludeFilter = x => x[0].toLowerCase().replaceAll(" ", "_").search(searchRegex) > -1; + tempResults = filteredWildcardsSorted.filter(x => baseFilter(x) || spaceIncludeFilter(x)) // Filter by tagword + + // Add final results + let finalResults = []; + tempResults.forEach(t => { + let result = new AutocompleteResult(t[0].trim(), ResultType.yamlWildcard) + result.count = t[1]; + finalResults.push(result); + }); + + return finalResults; + } else if (showAll) { + let filteredWildcardsSorted = filteredWildcards(""); + + // Add final results + let finalResults = []; + filteredWildcardsSorted.forEach(t => { + let result = new AutocompleteResult(t[0].trim(), ResultType.yamlWildcard) + result.count = t[1]; + finalResults.push(result); + }); + + originalTagword = tagword; + tagword = ""; + return finalResults; + } + } else { + let filteredWildcardsSorted = filteredWildcards(""); + + // Add final results + let finalResults = []; + filteredWildcardsSorted.forEach(t => { + let result = new AutocompleteResult(t[0].trim(), ResultType.yamlWildcard) + result.count = t[1]; + finalResults.push(result); + }); + + originalTagword = tagword; + tagword = ""; + return finalResults; + } + } +} + +PARSERS.push(new UmiParser(UMI_TRIGGER)); \ No newline at end of file diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 22c98e8..c904bc6 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -287,8 +287,7 @@ function isEnabled() { const WEIGHT_REGEX = /[([]([^,()[\]:| ]+)(?::(?:\d+(?:\.\d+)?|\.\d+))?[)\]]/g; const TAG_REGEX = /(<[^\t\n\r,>]+>?|[^\s,|<>]+|<)/g -const UMI_PROMPT_REGEX = /<[^\s]*?\[[^,<>]*[\]|]?>?/gi; -const UMI_TAG_REGEX = /(?:\[|\||--)([^<>\[\]\-|]+)/gi; + let hideBlocked = false; // On click, insert the tag into the prompt textbox with respect to the cursor position @@ -629,171 +628,7 @@ async function autocomplete(textArea, prompt, fixedTag = null) { results = results.sort((a, b) => a.text.localeCompare(b.text)); } } else { - if (CFG.useWildcards && [...tagword.matchAll(UMI_PROMPT_REGEX)].length > 0) { - // We are in a UMI yaml tag definition, parse further - let umiSubPrompts = [...prompt.matchAll(UMI_PROMPT_REGEX)]; - - let umiTags = []; - let umiTagsWithOperators = [] - - const insertAt = (str,char,pos) => str.slice(0,pos) + char + str.slice(pos); - - umiSubPrompts.forEach(umiSubPrompt => { - umiTags = umiTags.concat([...umiSubPrompt[0].matchAll(UMI_TAG_REGEX)].map(x => x[1].toLowerCase())); - - const start = umiSubPrompt.index; - const end = umiSubPrompt.index + umiSubPrompt[0].length; - if (textArea.selectionStart >= start && textArea.selectionStart <= end) { - umiTagsWithOperators = insertAt(umiSubPrompt[0], '###', textArea.selectionStart - start); - } - }); - - const promptSplitToTags = umiTagsWithOperators.replace(']###[', '][').split("]["); - - const clean = (str) => str - .replaceAll('>', '') - .replaceAll('<', '') - .replaceAll('[', '') - .replaceAll(']', '') - .trim(); - - const matches = promptSplitToTags.reduce((acc, curr) => { - isOptional = curr.includes("|"); - isNegative = curr.startsWith("--"); - let out; - if (isOptional) { - out = { - hasCursor: curr.includes("###"), - tags: clean(curr).split('|').map(x => ({ - hasCursor: x.includes("###"), - isNegative: x.startsWith("--"), - tag: clean(x).replaceAll("###", '').replaceAll("--", '') - })) - }; - acc.optional.push(out); - acc.all.push(...out.tags.map(x => x.tag)); - } else if (isNegative) { - out = { - hasCursor: curr.includes("###"), - tags: clean(curr).replaceAll("###", '').split('|'), - }; - out.tags = out.tags.map(x => x.startsWith("--") ? x.substring(2) : x); - acc.negative.push(out); - acc.all.push(...out.tags); - } else { - out = { - hasCursor: curr.includes("###"), - tags: clean(curr).replaceAll("###", '').split('|'), - }; - acc.positive.push(out); - acc.all.push(...out.tags); - } - return acc; - }, { positive: [], negative: [], optional: [], all: [] }); - - //console.log({ matches }) - - const filteredWildcards = (tagword) => { - const wildcards = yamlWildcards.filter(x => { - let tags = x[1]; - const matchesNeg = - matches.negative.length === 0 - || matches.negative.every(x => - x.hasCursor - || x.tags.every(t => !tags[t]) - ); - if (!matchesNeg) return false; - const matchesPos = - matches.positive.length === 0 - || matches.positive.every(x => - x.hasCursor - || x.tags.every(t => tags[t]) - ); - if (!matchesPos) return false; - const matchesOpt = - matches.optional.length === 0 - || matches.optional.some(x => - x.tags.some(t => - t.hasCursor - || t.isNegative - ? !tags[t.tag] - : tags[t.tag] - )); - if (!matchesOpt) return false; - return true; - }).reduce((acc, val) => { - Object.keys(val[1]).forEach(tag => acc[tag] = acc[tag] + 1 || 1); - return acc; - }, {}); - - return Object.entries(wildcards) - .sort((a, b) => b[1] - a[1]) - .filter(x => - x[0] === tagword - || !matches.all.includes(x[0]) - ); - } - - if (umiTags.length > 0) { - // Get difference for subprompt - let tagCountChange = umiTags.length - umiPreviousTags.length; - let diff = difference(umiTags, umiPreviousTags); - umiPreviousTags = umiTags; - - // Show all condition - let showAll = tagword.endsWith("[") || tagword.endsWith("[--") || tagword.endsWith("|"); - - // Exit early if the user closed the bracket manually - if ((!diff || diff.length === 0 || (diff.length === 1 && tagCountChange < 0)) && !showAll) { - if (!hideBlocked) hideResults(textArea); - return; - } - - let umiTagword = diff[0] || ''; - let tempResults = []; - if (umiTagword && umiTagword.length > 0) { - umiTagword = umiTagword.toLowerCase().replace(/[\n\r]/g, ""); - originalTagword = tagword; - tagword = umiTagword; - let filteredWildcardsSorted = filteredWildcards(umiTagword); - let searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(umiTagword)}`, 'i') - let baseFilter = x => x[0].toLowerCase().search(searchRegex) > -1; - let spaceIncludeFilter = x => x[0].toLowerCase().replaceAll(" ", "_").search(searchRegex) > -1; - tempResults = filteredWildcardsSorted.filter(x => baseFilter(x) || spaceIncludeFilter(x)) // Filter by tagword - - // Add final results - tempResults.forEach(t => { - let result = new AutocompleteResult(t[0].trim(), ResultType.yamlWildcard) - result.count = t[1]; - results.push(result); - }); - } else if (showAll) { - let filteredWildcardsSorted = filteredWildcards(""); - - // Add final results - filteredWildcardsSorted.forEach(t => { - let result = new AutocompleteResult(t[0].trim(), ResultType.yamlWildcard) - result.count = t[1]; - results.push(result); - }); - - originalTagword = tagword; - tagword = ""; - } - } else { - let filteredWildcardsSorted = filteredWildcards(""); - - // Add final results - filteredWildcardsSorted.forEach(t => { - let result = new AutocompleteResult(t[0].trim(), ResultType.yamlWildcard) - result.count = t[1]; - results.push(result); - }); - - originalTagword = tagword; - tagword = ""; - } - } else if (CFG.useEmbeddings && tagword.match(/ ]*>?/g)) { + if (CFG.useEmbeddings && tagword.match(/ ]*>?/g)) { // Show embeddings let tempResults = []; if (tagword !== " Date: Sun, 29 Jan 2023 17:19:24 +0100 Subject: [PATCH 13/26] Safety checks --- javascript/tagAutocomplete.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index c904bc6..545aeb1 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -409,6 +409,11 @@ function addResultsToList(textArea, results, tagword, resetList) { for (let i = resultCount; i < nextLength; i++) { let result = results[i]; + + // Skip if the result is null or undefined + if (!result) + continue; + let li = document.createElement("li"); let flexDiv = document.createElement("div"); @@ -803,7 +808,7 @@ async function autocomplete(textArea, prompt, fixedTag = null) { } // Guard for empty results - if (!results.length) { + if (!results || results.length === 0) { //console.log('No results found for "' + tagword + '"'); hideResults(textArea); return; From b0347d1ca74129e4dcfb9f25999c51254dac260a Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sun, 29 Jan 2023 17:36:02 +0100 Subject: [PATCH 14/26] Extract UMI after insert update --- javascript/ext_umi.js | 26 +++++++++++++++++++++++++- javascript/tagAutocomplete.js | 18 ++---------------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/javascript/ext_umi.js b/javascript/ext_umi.js index 0fe7fad..d66f592 100644 --- a/javascript/ext_umi.js +++ b/javascript/ext_umi.js @@ -23,6 +23,11 @@ class UmiParser extends BaseTagParser { } }); + // Safety check since UMI parsing sometimes seems to trigger outside of an UMI subprompt and thus fails + if (umiTagsWithOperators.length === 0) { + return null; + } + const promptSplitToTags = umiTagsWithOperators.replace(']###[', '][').split("]["); const clean = (str) => str @@ -178,4 +183,23 @@ class UmiParser extends BaseTagParser { } } -PARSERS.push(new UmiParser(UMI_TRIGGER)); \ No newline at end of file +function updateUmiTags(tagType, newPrompt, textArea) { + // If it was a yaml wildcard, also update the umiPreviousTags + if (tagType === ResultType.yamlWildcard && originalTagword.length > 0) { + let umiSubPrompts = [...newPrompt.matchAll(UMI_PROMPT_REGEX)]; + + let umiTags = []; + umiSubPrompts.forEach(umiSubPrompt => { + umiTags = umiTags.concat([...umiSubPrompt[0].matchAll(UMI_TAG_REGEX)].map(x => x[1].toLowerCase())); + }); + + umiPreviousTags = umiTags; + + hideResults(textArea); + } +} + +// Add UMI parser +PARSERS.push(new UmiParser(UMI_TRIGGER)); +// Add tag update after insert +QUEUE_AFTER_INSERT.push(updateUmiTags); \ No newline at end of file diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 545aeb1..406ca7d 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -361,21 +361,7 @@ function insertTextAtCursor(textArea, result, tagword) { previousTags = tags; // Callback - processQueue(QUEUE_AFTER_INSERT, null, tagType); - - // If it was a yaml wildcard, also update the umiPreviousTags - if (tagType === ResultType.yamlWildcard && originalTagword.length > 0) { - let umiSubPrompts = [...newPrompt.matchAll(UMI_PROMPT_REGEX)]; - - let umiTags = []; - umiSubPrompts.forEach(umiSubPrompt => { - umiTags = umiTags.concat([...umiSubPrompt[0].matchAll(UMI_TAG_REGEX)].map(x => x[1].toLowerCase())); - }); - - umiPreviousTags = umiTags; - - hideResults(textArea); - } + processQueue(QUEUE_AFTER_INSERT, null, tagType, newPrompt, textArea); // Hide results after inserting if (tagType === ResultType.wildcardFile) { @@ -383,7 +369,7 @@ function insertTextAtCursor(textArea, result, tagword) { hideBlocked = true; autocomplete(textArea, prompt, sanitizedText); setTimeout(() => { hideBlocked = false; }, 100); - } else { + } else if (!hideBlocked && isVisible(textArea)) { hideResults(textArea); } } From b22435dd32b78a9b11384406aa948525ebb46b32 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sun, 29 Jan 2023 17:58:21 +0100 Subject: [PATCH 15/26] Extract wildcard keep open as well --- javascript/__globals.js | 1 + javascript/ext_umi.js | 5 ++++- javascript/ext_wildcards.js | 16 +++++++++++++++- javascript/tagAutocomplete.js | 18 +++++++----------- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/javascript/__globals.js b/javascript/__globals.js index dacc065..7e62311 100644 --- a/javascript/__globals.js +++ b/javascript/__globals.js @@ -25,6 +25,7 @@ var resultCount = 0; var previousTags = []; var tagword = ""; var originalTagword = ""; +let hideBlocked = false; // Tag selection for keyboard navigation var selectedTag = null; diff --git a/javascript/ext_umi.js b/javascript/ext_umi.js index d66f592..5a4ed97 100644 --- a/javascript/ext_umi.js +++ b/javascript/ext_umi.js @@ -183,7 +183,7 @@ class UmiParser extends BaseTagParser { } } -function updateUmiTags(tagType, newPrompt, textArea) { +function updateUmiTags( tagType, sanitizedText, newPrompt, textArea) { // If it was a yaml wildcard, also update the umiPreviousTags if (tagType === ResultType.yamlWildcard && originalTagword.length > 0) { let umiSubPrompts = [...newPrompt.matchAll(UMI_PROMPT_REGEX)]; @@ -196,7 +196,10 @@ function updateUmiTags(tagType, newPrompt, textArea) { umiPreviousTags = umiTags; hideResults(textArea); + + return true; } + return false; } // Add UMI parser diff --git a/javascript/ext_wildcards.js b/javascript/ext_wildcards.js index 18143ef..8a2f205 100644 --- a/javascript/ext_wildcards.js +++ b/javascript/ext_wildcards.js @@ -55,6 +55,20 @@ class WildcardFileParser extends BaseTagParser { } } +function keepOpenIfWildcard(tagType, sanitizedText, newPrompt, textArea) { + // If it's a wildcard, we want to keep the results open so the user can select another wildcard + if (tagType === ResultType.wildcardFile) { + hideBlocked = true; + autocomplete(textArea, newPrompt, sanitizedText); + setTimeout(() => { hideBlocked = false; }, 100); + return true; + } + return false; +} + // Register the parsers PARSERS.push(new WildcardParser(WC_TRIGGER)); -PARSERS.push(new WildcardFileParser(WC_FILE_TRIGGER)); \ No newline at end of file +PARSERS.push(new WildcardFileParser(WC_FILE_TRIGGER)); + +// Add the keep open function to the queue +QUEUE_AFTER_INSERT.push(keepOpenIfWildcard); \ No newline at end of file diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 406ca7d..24acbdb 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -288,10 +288,8 @@ function isEnabled() { const WEIGHT_REGEX = /[([]([^,()[\]:| ]+)(?::(?:\d+(?:\.\d+)?|\.\d+))?[)\]]/g; const TAG_REGEX = /(<[^\t\n\r,>]+>?|[^\s,|<>]+|<)/g -let hideBlocked = false; - // On click, insert the tag into the prompt textbox with respect to the cursor position -function insertTextAtCursor(textArea, result, tagword) { +async function insertTextAtCursor(textArea, result, tagword) { let text = result.text; let tagType = result.type; @@ -361,15 +359,13 @@ function insertTextAtCursor(textArea, result, tagword) { previousTags = tags; // Callback - processQueue(QUEUE_AFTER_INSERT, null, tagType, newPrompt, textArea); + let returns = await processQueueReturn(QUEUE_AFTER_INSERT, null, tagType, sanitizedText, newPrompt, textArea); + // Return if any queue function returned true (has handled hide/show already) + if (returns.some(x => x === true)) + return; - // Hide results after inserting - if (tagType === ResultType.wildcardFile) { - // If it's a wildcard, we want to keep the results open so the user can select another wildcard - hideBlocked = true; - autocomplete(textArea, prompt, sanitizedText); - setTimeout(() => { hideBlocked = false; }, 100); - } else if (!hideBlocked && isVisible(textArea)) { + // Hide results after inserting, if it hasn't been hidden already by a queue function + if (!hideBlocked && isVisible(textArea)) { hideResults(textArea); } } From a588e0b9899abfa36cb9f42c39032e932328c36b Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sun, 29 Jan 2023 18:36:09 +0100 Subject: [PATCH 16/26] Extract embeddings, hypernets and loras --- javascript/ext_embeddings.js | 35 ++++++ javascript/ext_hypernets.js | 27 ++++ javascript/ext_loras.js | 27 ++++ javascript/tagAutocomplete.js | 223 +++++++++------------------------- 4 files changed, 146 insertions(+), 166 deletions(-) create mode 100644 javascript/ext_embeddings.js create mode 100644 javascript/ext_hypernets.js create mode 100644 javascript/ext_loras.js diff --git a/javascript/ext_embeddings.js b/javascript/ext_embeddings.js new file mode 100644 index 0000000..2d952a2 --- /dev/null +++ b/javascript/ext_embeddings.js @@ -0,0 +1,35 @@ +const EMB_REGEX = /<(?!l:|h:)[^,> ]*>?/g; +const EMB_TRIGGER = () => CFG.useEmbeddings && tagword.match(EMB_REGEX); + +class EmbeddingParser extends BaseTagParser { + parse() { + // Show embeddings + let tempResults = []; + if (tagword !== "<" && tagword !== " x[0].toLowerCase().includes(searchTerm) && x[1] && x[1] === versionString); // Filter by tagword + else + tempResults = embeddings.filter(x => x[0].toLowerCase().includes(searchTerm)); // Filter by tagword + } else { + tempResults = embeddings; + } + + // Add final results + let finalResults = []; + tempResults.forEach(t => { + let result = new AutocompleteResult(t[0].trim(), ResultType.embedding) + result.meta = t[1] + " Embedding"; + finalResults.push(result); + }); + + return finalResults; + } +} + +PARSERS.push(new EmbeddingParser(EMB_TRIGGER)); \ No newline at end of file diff --git a/javascript/ext_hypernets.js b/javascript/ext_hypernets.js new file mode 100644 index 0000000..221dcb3 --- /dev/null +++ b/javascript/ext_hypernets.js @@ -0,0 +1,27 @@ +const HYP_REGEX = /<(?!e:|l:)[^,> ]*>?/g; +const HYP_TRIGGER = () => CFG.useHypernetworks && tagword.match(HYP_REGEX); + +class HypernetParser extends BaseTagParser { + parse() { + // Show hypernetworks + let tempResults = []; + if (tagword !== "<" && tagword !== " x.toLowerCase().includes(searchTerm)); // Filter by tagword + } else { + tempResults = hypernetworks; + } + + // Add final results + let finalResults = []; + tempResults.forEach(t => { + let result = new AutocompleteResult(t.trim(), ResultType.hypernetwork) + result.meta = "Hypernetwork"; + finalResults.push(result); + }); + + return finalResults; + } +} + +PARSERS.push(new HypernetParser(HYP_TRIGGER)); \ No newline at end of file diff --git a/javascript/ext_loras.js b/javascript/ext_loras.js new file mode 100644 index 0000000..82c4012 --- /dev/null +++ b/javascript/ext_loras.js @@ -0,0 +1,27 @@ +const LORA_REGEX = /<(?!e:|h:)[^,> ]*>?/g; +const LORA_TRIGGER = () => CFG.useLoras && tagword.match(LORA_REGEX); + +class LoraParser extends BaseTagParser { + parse() { + // Show lora + let tempResults = []; + if (tagword !== "<" && tagword !== " x.toLowerCase().includes(searchTerm)); // Filter by tagword + } else { + tempResults = loras; + } + + // Add final results + let finalResults = []; + tempResults.forEach(t => { + let result = new AutocompleteResult(t.trim(), ResultType.lora) + result.meta = "Lora"; + finalResults.push(result); + }); + + return finalResults; + } +} + +PARSERS.push(new LoraParser(LORA_TRIGGER)); \ No newline at end of file diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 24acbdb..5b12fc6 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -605,6 +605,7 @@ async function autocomplete(textArea, prompt, fixedTag = null) { // Process all parsers let resultCandidates = await processParsers(textArea, prompt); + // If one ore more result candidates match, use their results if (resultCandidates && resultCandidates.length > 0) { // Flatten our candidate(s) results = resultCandidates.flat(); @@ -613,179 +614,69 @@ async function autocomplete(textArea, prompt, fixedTag = null) { let shouldSort = resultCandidates.length > 1; if (shouldSort) { results = results.sort((a, b) => a.text.localeCompare(b.text)); - } - } else { - if (CFG.useEmbeddings && tagword.match(/ ]*>?/g)) { - // Show embeddings - let tempResults = []; - if (tagword !== " x[0].toLowerCase().includes(searchTerm) && x[1] && x[1] === versionString); // Filter by tagword - else - tempResults = embeddings.filter(x => x[0].toLowerCase().includes(searchTerm)); // Filter by tagword - } else { - tempResults = embeddings; - } - // Add final results - tempResults.forEach(t => { - let result = new AutocompleteResult(t[0].trim(), ResultType.embedding) - result.meta = t[1] + " Embedding"; - results.push(result); - }); - } else if(CFG.useHypernetworks && tagword.match(/ ]*>?/g)) { - // Show hypernetworks - let tempResults = []; - if (tagword !== " x.toLowerCase().includes(searchTerm)); // Filter by tagword - } else { - tempResults = hypernetworks; - } - - // Add final results - tempResults.forEach(t => { - let result = new AutocompleteResult(t.trim(), ResultType.hypernetwork) - result.meta = "Hypernetwork"; - results.push(result); - }); - } else if(CFG.useLoras && tagword.match(/ ]*>?/g)){ - // Show lora - let tempResults = []; - if (tagword !== " x.toLowerCase().includes(searchTerm)); // Filter by tagword - } else { - tempResults = loras; - } - - // Add final results - tempResults.forEach(t => { - let result = new AutocompleteResult(t.trim(), ResultType.lora) - result.meta = "Lora"; - results.push(result); - }); - } else if ((CFG.useEmbeddings || CFG.useHypernetworks || CFG.useLoras) && tagword.match(/<[^,> ]*>?/g)) { - // Embeddings, lora, wildcards all together with generic options - let tempEmbResults = []; - let tempHypResults = []; - let tempLoraResults = []; - if (tagword !== "<") { - let searchTerm = tagword.replace("<", "") - - let versionString; - if (searchTerm.startsWith("v1") || searchTerm.startsWith("v2")) { - versionString = searchTerm.slice(0, 2); - searchTerm = searchTerm.slice(2); - } - - if (versionString && CFG.useEmbeddings) { - // Version string is only for embeddings atm, so we don't search the other lists here. - tempEmbResults = embeddings.filter(x => x[0].toLowerCase().includes(searchTerm) && x[1] && x[1] === versionString); // Filter by tagword + // Since some tags are kaomoji, we have to add the normal results in some cases + if (tagword.startsWith("<") || tagword.startsWith("*<")) { + // Create escaped search regex with support for * as a start placeholder + let searchRegex; + if (tagword.startsWith("*")) { + tagword = tagword.slice(1); + searchRegex = new RegExp(`${escapeRegExp(tagword)}`, 'i'); } else { - tempEmbResults = embeddings.filter(x => x[0].toLowerCase().includes(searchTerm)); // Filter by tagword - tempHypResults = hypernetworks.filter(x => x.toLowerCase().includes(searchTerm)); // Filter by tagword - tempLoraResults = loras.filter(x => x.toLowerCase().includes(searchTerm)); // Filter by tagword + searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(tagword)}`, 'i'); } - } else { - tempEmbResults = embeddings; - tempHypResults = hypernetworks; - tempLoraResults = loras; - } + let genericResults = allTags.filter(x => x[0].toLowerCase().search(searchRegex) > -1).slice(0, CFG.maxResults); - // Since some tags are kaomoji, we have to still get the normal results first. - // Create escaped search regex with support for * as a start placeholder - let searchRegex; - if (tagword.startsWith("*")) { - tagword = tagword.slice(1); - searchRegex = new RegExp(`${escapeRegExp(tagword)}`, 'i'); - } else { - searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(tagword)}`, 'i'); - } - let genericResults = allTags.filter(x => x[0].toLowerCase().search(searchRegex) > -1).slice(0, CFG.maxResults); - - // Add final results - let mixedResults = []; - if (CFG.useEmbeddings) { - tempEmbResults.forEach(t => { - let result = new AutocompleteResult(t[0].trim(), ResultType.embedding) - result.meta = t[1] + " Embedding"; - mixedResults.push(result); - }); - } - if (CFG.useHypernetworks) { - tempHypResults.forEach(t => { - let result = new AutocompleteResult(t.trim(), ResultType.hypernetwork) - result.meta = "Hypernetwork"; - mixedResults.push(result); - }); - } - if (CFG.useLoras) { - tempLoraResults.forEach(t => { - let result = new AutocompleteResult(t.trim(), ResultType.lora) - result.meta = "Lora"; - mixedResults.push(result); - }); - } - - // Add all mixed results to the final results, sorted by name so that they aren't after one another. - results = mixedResults.sort((a, b) => a.text.localeCompare(b.text)); - - genericResults.forEach(g => { - let result = new AutocompleteResult(g[0].trim(), ResultType.tag) - result.category = g[1]; - result.count = g[2]; - result.aliases = g[3]; - results.push(result); - }); - } else { - // Create escaped search regex with support for * as a start placeholder - let searchRegex; - if (tagword.startsWith("*")) { - tagword = tagword.slice(1); - searchRegex = new RegExp(`${escapeRegExp(tagword)}`, 'i'); - } else { - searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(tagword)}`, 'i'); - } - // If onlyShowAlias is enabled, we don't need to include normal results - if (CFG.alias.onlyShowAlias) { - results = allTags.filter(x => x[3] && x[3].toLowerCase().search(searchRegex) > -1); - } else { - // Else both normal tags and aliases/translations are included depending on the config - let baseFilter = (x) => x[0].toLowerCase().search(searchRegex) > -1; - let aliasFilter = (x) => x[3] && x[3].toLowerCase().search(searchRegex) > -1; - let translationFilter = (x) => (translations.has(x[0]) && translations.get(x[0]).toLowerCase().search(searchRegex) > -1) - || x[3] && x[3].split(",").some(y => translations.has(y) && translations.get(y).toLowerCase().search(searchRegex) > -1); - - let fil; - if (CFG.alias.searchByAlias && CFG.translation.searchByTranslation) - fil = (x) => baseFilter(x) || aliasFilter(x) || translationFilter(x); - else if (CFG.alias.searchByAlias && !CFG.translation.searchByTranslation) - fil = (x) => baseFilter(x) || aliasFilter(x); - else if (CFG.translation.searchByTranslation && !CFG.alias.searchByAlias) - fil = (x) => baseFilter(x) || translationFilter(x); - else - fil = (x) => baseFilter(x); - - // Add final results - allTags.filter(fil).forEach(t => { - let result = new AutocompleteResult(t[0].trim(), ResultType.tag) - result.category = t[1]; - result.count = t[2]; - result.aliases = t[3]; + genericResults.forEach(g => { + let result = new AutocompleteResult(g[0].trim(), ResultType.tag) + result.category = g[1]; + result.count = g[2]; + result.aliases = g[3]; results.push(result); }); } - // Slice if the user has set a max result count - if (!CFG.showAllResults) { - results = results.slice(0, CFG.maxResults); - } + } + } else { // Else search the normal tag list + // Create escaped search regex with support for * as a start placeholder + let searchRegex; + if (tagword.startsWith("*")) { + tagword = tagword.slice(1); + searchRegex = new RegExp(`${escapeRegExp(tagword)}`, 'i'); + } else { + searchRegex = new RegExp(`(^|[^a-zA-Z])${escapeRegExp(tagword)}`, 'i'); + } + // If onlyShowAlias is enabled, we don't need to include normal results + if (CFG.alias.onlyShowAlias) { + results = allTags.filter(x => x[3] && x[3].toLowerCase().search(searchRegex) > -1); + } else { + // Else both normal tags and aliases/translations are included depending on the config + let baseFilter = (x) => x[0].toLowerCase().search(searchRegex) > -1; + let aliasFilter = (x) => x[3] && x[3].toLowerCase().search(searchRegex) > -1; + let translationFilter = (x) => (translations.has(x[0]) && translations.get(x[0]).toLowerCase().search(searchRegex) > -1) + || x[3] && x[3].split(",").some(y => translations.has(y) && translations.get(y).toLowerCase().search(searchRegex) > -1); + + let fil; + if (CFG.alias.searchByAlias && CFG.translation.searchByTranslation) + fil = (x) => baseFilter(x) || aliasFilter(x) || translationFilter(x); + else if (CFG.alias.searchByAlias && !CFG.translation.searchByTranslation) + fil = (x) => baseFilter(x) || aliasFilter(x); + else if (CFG.translation.searchByTranslation && !CFG.alias.searchByAlias) + fil = (x) => baseFilter(x) || translationFilter(x); + else + fil = (x) => baseFilter(x); + + // Add final results + allTags.filter(fil).forEach(t => { + let result = new AutocompleteResult(t[0].trim(), ResultType.tag) + result.category = t[1]; + result.count = t[2]; + result.aliases = t[3]; + results.push(result); + }); + } + // Slice if the user has set a max result count + if (!CFG.showAllResults) { + results = results.slice(0, CFG.maxResults); } } From db3319b0d3baae916099ab9b4f840bd20040f8a6 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sun, 29 Jan 2023 18:43:09 +0100 Subject: [PATCH 17/26] Fix long lists not scrolling to top on reset --- javascript/tagAutocomplete.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 5b12fc6..ee94115 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -524,6 +524,9 @@ function addResultsToList(textArea, results, tagword, resetList) { resultsList.appendChild(li); } resultCount = nextLength; + + if (resetList) + resultDiv.scrollTop = 0; } function updateSelectionStyle(textArea, newIndex, oldIndex) { From c70a18919b146bf70ad96e8abf5aa43b88b7dd37 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Thu, 2 Feb 2023 18:56:07 +0100 Subject: [PATCH 18/26] Make tag regex work with more < configurations Will now allow completion of a < tag if the one directly after is also a < tag only separated by a space. (Happens often now that Loras are a thing and <>'s stay in the prompt with them) --- javascript/tagAutocomplete.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index ee94115..cacb048 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -286,7 +286,9 @@ function isEnabled() { } const WEIGHT_REGEX = /[([]([^,()[\]:| ]+)(?::(?:\d+(?:\.\d+)?|\.\d+))?[)\]]/g; -const TAG_REGEX = /(<[^\t\n\r,>]+>?|[^\s,|<>]+|<)/g +const POINTY_REGEX = /<[^\s,<](?:[^\t\n\r,<>]*>|[^\t\n\r,> ]*)/g; +const NORMAL_TAG_REGEX = /[^\s,|<>]+| Date: Thu, 2 Feb 2023 18:56:52 +0100 Subject: [PATCH 19/26] Add pycache folder to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e15d601..e9e3707 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ tags/temp/ +__pycache__/ From cbeced9121286294bbd8842146c3f8a74ac6b4b4 Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Fri, 10 Feb 2023 11:55:56 +0100 Subject: [PATCH 20/26] Extract file load to queue This enables other parsers to keep their load function in the same file --- javascript/__globals.js | 1 + javascript/_utils.js | 4 +- javascript/ext_embeddings.js | 17 ++++++- javascript/ext_hypernets.js | 17 ++++++- javascript/ext_loras.js | 17 ++++++- javascript/ext_umi.js | 24 ++++++++- javascript/ext_wildcards.js | 42 +++++++++++++++- javascript/tagAutocomplete.js | 91 ++--------------------------------- 8 files changed, 119 insertions(+), 94 deletions(-) diff --git a/javascript/__globals.js b/javascript/__globals.js index 7e62311..af572be 100644 --- a/javascript/__globals.js +++ b/javascript/__globals.js @@ -42,6 +42,7 @@ var umiPreviousTags = []; // Queues const QUEUE_AFTER_INSERT = []; const QUEUE_AFTER_SETUP = []; +const QUEUE_FILE_LOAD = []; const QUEUE_AFTER_CONFIG_CHANGE = []; // List of parsers to try diff --git a/javascript/_utils.js b/javascript/_utils.js index a8251b9..06810db 100644 --- a/javascript/_utils.js +++ b/javascript/_utils.js @@ -96,9 +96,9 @@ function escapeHTML(unsafeText) { } // Queue calling function to process global queues -function processQueue(queue, context, ...args) { +async function processQueue(queue, context, ...args) { for (let i = 0; i < queue.length; i++) { - queue[i].call(context, ...args); + await queue[i].call(context, ...args); } } // The same but with return values diff --git a/javascript/ext_embeddings.js b/javascript/ext_embeddings.js index 2d952a2..ca83c9e 100644 --- a/javascript/ext_embeddings.js +++ b/javascript/ext_embeddings.js @@ -32,4 +32,19 @@ class EmbeddingParser extends BaseTagParser { } } -PARSERS.push(new EmbeddingParser(EMB_TRIGGER)); \ No newline at end of file +async function load() { + if (embeddings.length === 0) { + try { + embeddings = (await readFile(`${tagBasePath}/temp/emb.txt?${new Date().getTime()}`)).split("\n") + .filter(x => x.trim().length > 0) // Remove empty lines + .map(x => x.trim().split(",")); // Split into name, version type pairs + } catch (e) { + console.error("Error loading embeddings.txt: " + e); + } + } +} + +PARSERS.push(new EmbeddingParser(EMB_TRIGGER)); + +// Add load function to the queue +QUEUE_FILE_LOAD.push(load); \ No newline at end of file diff --git a/javascript/ext_hypernets.js b/javascript/ext_hypernets.js index 221dcb3..8e011cf 100644 --- a/javascript/ext_hypernets.js +++ b/javascript/ext_hypernets.js @@ -24,4 +24,19 @@ class HypernetParser extends BaseTagParser { } } -PARSERS.push(new HypernetParser(HYP_TRIGGER)); \ No newline at end of file +async function load() { + if (hypernetworks.length === 0) { + try { + hypernetworks = (await readFile(`${tagBasePath}/temp/hyp.txt?${new Date().getTime()}`)).split("\n") + .filter(x => x.trim().length > 0) //Remove empty lines + .map(x => x.trim()); // Remove carriage returns and padding if it exists + } catch (e) { + console.error("Error loading hypernetworks.txt: " + e); + } + } +} + +PARSERS.push(new HypernetParser(HYP_TRIGGER)); + +// Add load function to the queue +QUEUE_FILE_LOAD.push(load); \ No newline at end of file diff --git a/javascript/ext_loras.js b/javascript/ext_loras.js index 82c4012..9279185 100644 --- a/javascript/ext_loras.js +++ b/javascript/ext_loras.js @@ -24,4 +24,19 @@ class LoraParser extends BaseTagParser { } } -PARSERS.push(new LoraParser(LORA_TRIGGER)); \ No newline at end of file +async function load() { + if (loras.length === 0) { + try { + loras = (await readFile(`${tagBasePath}/temp/lora.txt?${new Date().getTime()}`)).split("\n") + .filter(x => x.trim().length > 0) // Remove empty lines + .map(x => x.trim()); // Remove carriage returns and padding if it exists + } catch (e) { + console.error("Error loading lora.txt: " + e); + } + } +} + +PARSERS.push(new LoraParser(LORA_TRIGGER)); + +// Add load function to the queue +QUEUE_FILE_LOAD.push(load); \ No newline at end of file diff --git a/javascript/ext_umi.js b/javascript/ext_umi.js index 5a4ed97..c476e7b 100644 --- a/javascript/ext_umi.js +++ b/javascript/ext_umi.js @@ -202,7 +202,29 @@ function updateUmiTags( tagType, sanitizedText, newPrompt, textArea) { return false; } +async function load() { + if (yamlWildcards.length === 0) { + try { + let yamlTags = (await readFile(`${tagBasePath}/temp/wcet.txt?${new Date().getTime()}`)).split("\n"); + // Split into tag, count pairs + yamlWildcards = yamlTags.map(x => x + .trim() + .split(",")) + .map(([i, ...rest]) => [ + i, + rest.reduce((a, b) => { + a[b.toLowerCase()] = true; + return a; + }, {}), + ]); + } catch (e) { + console.error("Error loading yaml wildcards: " + e); + } + } +} + // Add UMI parser PARSERS.push(new UmiParser(UMI_TRIGGER)); // Add tag update after insert -QUEUE_AFTER_INSERT.push(updateUmiTags); \ No newline at end of file +QUEUE_AFTER_INSERT.push(updateUmiTags); +QUEUE_FILE_LOAD.push(load); \ No newline at end of file diff --git a/javascript/ext_wildcards.js b/javascript/ext_wildcards.js index 8a2f205..f976449 100644 --- a/javascript/ext_wildcards.js +++ b/javascript/ext_wildcards.js @@ -55,6 +55,44 @@ class WildcardFileParser extends BaseTagParser { } } +async function load() { + if (wildcardFiles.length === 0 && wildcardExtFiles.length === 0) { + try { + let wcFileArr = (await readFile(`${tagBasePath}/temp/wc.txt?${new Date().getTime()}`)).split("\n"); + let wcBasePath = wcFileArr[0].trim(); // First line should be the base path + wildcardFiles = wcFileArr.slice(1) + .filter(x => x.trim().length > 0) // Remove empty lines + .map(x => [wcBasePath, x.trim().replace(".txt", "")]); // Remove file extension & newlines + + // To support multiple sources, we need to separate them using the provided "-----" strings + let wcExtFileArr = (await readFile(`${tagBasePath}/temp/wce.txt?${new Date().getTime()}`)).split("\n"); + let splitIndices = []; + for (let index = 0; index < wcExtFileArr.length; index++) { + if (wcExtFileArr[index].trim() === "-----") { + splitIndices.push(index); + } + } + // For each group, add them to the wildcardFiles array with the base path as the first element + for (let i = 0; i < splitIndices.length; i++) { + let start = splitIndices[i - 1] || 0; + if (i > 0) start++; // Skip the "-----" line + let end = splitIndices[i]; + + let wcExtFile = wcExtFileArr.slice(start, end); + let base = wcExtFile[0].trim() + "/"; + wcExtFile = wcExtFile.slice(1) + .filter(x => x.trim().length > 0) // Remove empty lines + .map(x => x.trim().replace(base, "").replace(".txt", "")); // Remove file extension & newlines; + + wcExtFile = wcExtFile.map(x => [base, x]); + wildcardExtFiles.push(...wcExtFile); + } + } catch (e) { + console.error("Error loading wildcards: " + e); + } + } +} + function keepOpenIfWildcard(tagType, sanitizedText, newPrompt, textArea) { // If it's a wildcard, we want to keep the results open so the user can select another wildcard if (tagType === ResultType.wildcardFile) { @@ -71,4 +109,6 @@ PARSERS.push(new WildcardParser(WC_TRIGGER)); PARSERS.push(new WildcardFileParser(WC_FILE_TRIGGER)); // Add the keep open function to the queue -QUEUE_AFTER_INSERT.push(keepOpenIfWildcard); \ No newline at end of file +QUEUE_AFTER_INSERT.push(keepOpenIfWildcard); +// Add the load function to the queue +QUEUE_FILE_LOAD.push(load); \ No newline at end of file diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index cacb048..89007b2 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -226,7 +226,7 @@ async function syncOptions() { CFG = newCFG; // Callback - processQueue(QUEUE_AFTER_CONFIG_CHANGE, null); + await processQueue(QUEUE_AFTER_CONFIG_CHANGE, null); } // Create the result list div and necessary styling @@ -783,91 +783,8 @@ async function setup() { // Load colors CFG["colors"] = (await readFile(`${tagBasePath}/colors.json?${new Date().getTime()}`, true)); - // Load wildcards - if (wildcardFiles.length === 0 && wildcardExtFiles.length === 0) { - try { - let wcFileArr = (await readFile(`${tagBasePath}/temp/wc.txt?${new Date().getTime()}`)).split("\n"); - let wcBasePath = wcFileArr[0].trim(); // First line should be the base path - wildcardFiles = wcFileArr.slice(1) - .filter(x => x.trim().length > 0) // Remove empty lines - .map(x => [wcBasePath, x.trim().replace(".txt", "")]); // Remove file extension & newlines - - // To support multiple sources, we need to separate them using the provided "-----" strings - let wcExtFileArr = (await readFile(`${tagBasePath}/temp/wce.txt?${new Date().getTime()}`)).split("\n"); - let splitIndices = []; - for (let index = 0; index < wcExtFileArr.length; index++) { - if (wcExtFileArr[index].trim() === "-----") { - splitIndices.push(index); - } - } - // For each group, add them to the wildcardFiles array with the base path as the first element - for (let i = 0; i < splitIndices.length; i++) { - let start = splitIndices[i - 1] || 0; - if (i > 0) start++; // Skip the "-----" line - let end = splitIndices[i]; - - let wcExtFile = wcExtFileArr.slice(start, end); - let base = wcExtFile[0].trim() + "/"; - wcExtFile = wcExtFile.slice(1) - .filter(x => x.trim().length > 0) // Remove empty lines - .map(x => x.trim().replace(base, "").replace(".txt", "")); // Remove file extension & newlines; - - wcExtFile = wcExtFile.map(x => [base, x]); - wildcardExtFiles.push(...wcExtFile); - } - } catch (e) { - console.error("Error loading wildcards: " + e); - } - } - // Load yaml wildcards - if (yamlWildcards.length === 0) { - try { - let yamlTags = (await readFile(`${tagBasePath}/temp/wcet.txt?${new Date().getTime()}`)).split("\n"); - // Split into tag, count pairs - yamlWildcards = yamlTags.map(x => x - .trim() - .split(",")) - .map(([i, ...rest]) => [ - i, - rest.reduce((a, b) => { - a[b.toLowerCase()] = true; - return a; - }, {}), - ]); - } catch (e) { - console.error("Error loading yaml wildcards: " + e); - } - } - // Load embeddings - if (embeddings.length === 0) { - try { - embeddings = (await readFile(`${tagBasePath}/temp/emb.txt?${new Date().getTime()}`)).split("\n") - .filter(x => x.trim().length > 0) // Remove empty lines - .map(x => x.trim().split(",")); // Split into name, version type pairs - } catch (e) { - console.error("Error loading embeddings.txt: " + e); - } - } - // Load hypernetworks - if (hypernetworks.length === 0) { - try { - hypernetworks = (await readFile(`${tagBasePath}/temp/hyp.txt?${new Date().getTime()}`)).split("\n") - .filter(x => x.trim().length > 0) //Remove empty lines - .map(x => x.trim()); // Remove carriage returns and padding if it exists - } catch (e) { - console.error("Error loading hypernetworks.txt: " + e); - } - } - // Load lora - if (loras.length === 0) { - try { - loras = (await readFile(`${tagBasePath}/temp/lora.txt?${new Date().getTime()}`)).split("\n") - .filter(x => x.trim().length > 0) // Remove empty lines - .map(x => x.trim()); // Remove carriage returns and padding if it exists - } catch (e) { - console.error("Error loading lora.txt: " + e); - } - } + // Load external files needed by completion extensions + await processQueue(QUEUE_FILE_LOAD, null); // Find all textareas let textAreas = getTextAreas(); @@ -984,7 +901,7 @@ async function setup() { gradioApp().appendChild(acStyle); // Callback - processQueue(QUEUE_AFTER_SETUP, null); + await processQueue(QUEUE_AFTER_SETUP, null); } onUiUpdate(async () => { From d29298e0cc8bb6f6ea1b9497957a6451b61e9b6c Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Fri, 10 Feb 2023 11:59:06 +0100 Subject: [PATCH 21/26] Move anti-caching parameter to load function For less repetition and shorter paths in the higher level functions. Active by default, but can be disabled. --- javascript/_utils.js | 5 ++++- javascript/ext_embeddings.js | 2 +- javascript/ext_hypernets.js | 2 +- javascript/ext_loras.js | 2 +- javascript/ext_umi.js | 2 +- javascript/ext_wildcards.js | 6 +++--- javascript/tagAutocomplete.js | 10 +++++----- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/javascript/_utils.js b/javascript/_utils.js index 06810db..863b4d3 100644 --- a/javascript/_utils.js +++ b/javascript/_utils.js @@ -38,7 +38,10 @@ function parseCSV(str) { } // Load file -async function readFile(filePath, json = false) { +async function readFile(filePath, json = false, cache = false) { + if (!cache) + filePath += `?${new Date().getTime()}`; + let response = await fetch(`file=${filePath}`); if (response.status != 200) { diff --git a/javascript/ext_embeddings.js b/javascript/ext_embeddings.js index ca83c9e..15070b7 100644 --- a/javascript/ext_embeddings.js +++ b/javascript/ext_embeddings.js @@ -35,7 +35,7 @@ class EmbeddingParser extends BaseTagParser { async function load() { if (embeddings.length === 0) { try { - embeddings = (await readFile(`${tagBasePath}/temp/emb.txt?${new Date().getTime()}`)).split("\n") + embeddings = (await readFile(`${tagBasePath}/temp/emb.txt`)).split("\n") .filter(x => x.trim().length > 0) // Remove empty lines .map(x => x.trim().split(",")); // Split into name, version type pairs } catch (e) { diff --git a/javascript/ext_hypernets.js b/javascript/ext_hypernets.js index 8e011cf..9b5dd4c 100644 --- a/javascript/ext_hypernets.js +++ b/javascript/ext_hypernets.js @@ -27,7 +27,7 @@ class HypernetParser extends BaseTagParser { async function load() { if (hypernetworks.length === 0) { try { - hypernetworks = (await readFile(`${tagBasePath}/temp/hyp.txt?${new Date().getTime()}`)).split("\n") + hypernetworks = (await readFile(`${tagBasePath}/temp/hyp.txt`)).split("\n") .filter(x => x.trim().length > 0) //Remove empty lines .map(x => x.trim()); // Remove carriage returns and padding if it exists } catch (e) { diff --git a/javascript/ext_loras.js b/javascript/ext_loras.js index 9279185..a9b7051 100644 --- a/javascript/ext_loras.js +++ b/javascript/ext_loras.js @@ -27,7 +27,7 @@ class LoraParser extends BaseTagParser { async function load() { if (loras.length === 0) { try { - loras = (await readFile(`${tagBasePath}/temp/lora.txt?${new Date().getTime()}`)).split("\n") + loras = (await readFile(`${tagBasePath}/temp/lora.txt`)).split("\n") .filter(x => x.trim().length > 0) // Remove empty lines .map(x => x.trim()); // Remove carriage returns and padding if it exists } catch (e) { diff --git a/javascript/ext_umi.js b/javascript/ext_umi.js index c476e7b..9384e97 100644 --- a/javascript/ext_umi.js +++ b/javascript/ext_umi.js @@ -205,7 +205,7 @@ function updateUmiTags( tagType, sanitizedText, newPrompt, textArea) { async function load() { if (yamlWildcards.length === 0) { try { - let yamlTags = (await readFile(`${tagBasePath}/temp/wcet.txt?${new Date().getTime()}`)).split("\n"); + let yamlTags = (await readFile(`${tagBasePath}/temp/wcet.txt`)).split("\n"); // Split into tag, count pairs yamlWildcards = yamlTags.map(x => x .trim() diff --git a/javascript/ext_wildcards.js b/javascript/ext_wildcards.js index f976449..22a2944 100644 --- a/javascript/ext_wildcards.js +++ b/javascript/ext_wildcards.js @@ -17,7 +17,7 @@ class WildcardParser extends BaseTagParser { // Use found wildcard file or look in external wildcard files let wcPair = wcFound || wildcardExtFiles.find(x => x[1].toLowerCase() === wcFile); - let wildcards = (await readFile(`${wcPair[0]}/${wcPair[1]}.txt?${new Date().getTime()}`)).split("\n") + let wildcards = (await readFile(`${wcPair[0]}/${wcPair[1]}.txt`)).split("\n") .filter(x => x.trim().length > 0 && !x.startsWith('#')); // Remove empty lines and comments let finalResults = []; @@ -58,14 +58,14 @@ class WildcardFileParser extends BaseTagParser { async function load() { if (wildcardFiles.length === 0 && wildcardExtFiles.length === 0) { try { - let wcFileArr = (await readFile(`${tagBasePath}/temp/wc.txt?${new Date().getTime()}`)).split("\n"); + let wcFileArr = (await readFile(`${tagBasePath}/temp/wc.txt`)).split("\n"); let wcBasePath = wcFileArr[0].trim(); // First line should be the base path wildcardFiles = wcFileArr.slice(1) .filter(x => x.trim().length > 0) // Remove empty lines .map(x => [wcBasePath, x.trim().replace(".txt", "")]); // Remove file extension & newlines // To support multiple sources, we need to separate them using the provided "-----" strings - let wcExtFileArr = (await readFile(`${tagBasePath}/temp/wce.txt?${new Date().getTime()}`)).split("\n"); + let wcExtFileArr = (await readFile(`${tagBasePath}/temp/wce.txt`)).split("\n"); let splitIndices = []; for (let index = 0; index < wcExtFileArr.length; index++) { if (wcExtFileArr[index].trim() === "-----") { diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 89007b2..d6b54dc 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -87,14 +87,14 @@ async function loadTags(c) { // Load main tags and aliases if (allTags.length === 0 && c.tagFile && c.tagFile !== "None") { try { - allTags = await loadCSV(`${tagBasePath}/${c.tagFile}?${new Date().getTime()}`); + allTags = await loadCSV(`${tagBasePath}/${c.tagFile}`); } catch (e) { console.error("Error loading tags file: " + e); return; } if (c.extra.extraFile && c.extra.extraFile !== "None") { try { - extras = await loadCSV(`${tagBasePath}/${c.extra.extraFile}?${new Date().getTime()}`); + extras = await loadCSV(`${tagBasePath}/${c.extra.extraFile}`); if (c.extra.onlyAliasExtraFile) { // This works purely on index, so it's not very robust. But a lot faster. for (let i = 0, n = extras.length; i < n; i++) { @@ -135,7 +135,7 @@ async function loadTags(c) { async function loadTranslations(c) { if (c.translation.translationFile && c.translation.translationFile !== "None") { try { - let tArray = await loadCSV(`${tagBasePath}/${c.translation.translationFile}?${new Date().getTime()}`); + let tArray = await loadCSV(`${tagBasePath}/${c.translation.translationFile}`); tArray.forEach(t => { if (c.translation.oldFormat) translations.set(t[0], t[2]); @@ -781,7 +781,7 @@ function navigateInList(textArea, event) { // One-time setup, triggered from onUiUpdate async function setup() { // Load colors - CFG["colors"] = (await readFile(`${tagBasePath}/colors.json?${new Date().getTime()}`, true)); + CFG["colors"] = (await readFile(`${tagBasePath}/colors.json`, true)); // Load external files needed by completion extensions await processQueue(QUEUE_FILE_LOAD, null); @@ -909,7 +909,7 @@ onUiUpdate(async () => { if (CFG) return; // Get our tag base path from the temp file - tagBasePath = await readFile(`tmp/tagAutocompletePath.txt?${new Date().getTime()}`); + tagBasePath = await readFile(`tmp/tagAutocompletePath.txt`); // Load config from webui opts await syncOptions(); // Rest of setup From b5404001104e550a7b955655583b9ed25bbe15fe Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Fri, 10 Feb 2023 12:23:52 +0100 Subject: [PATCH 22/26] Allow spaces in wildcard file names --- javascript/ext_wildcards.js | 2 +- javascript/tagAutocomplete.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/javascript/ext_wildcards.js b/javascript/ext_wildcards.js index 22a2944..b4eae15 100644 --- a/javascript/ext_wildcards.js +++ b/javascript/ext_wildcards.js @@ -1,5 +1,5 @@ // Regex -const WC_REGEX = /\b__([^, ]+)__([^, ]*)\b/g; +const WC_REGEX = /\b__([^,]+)__([^, ]*)\b/g; // Trigger conditions const WC_TRIGGER = () => CFG.useWildcards && [...tagword.matchAll(WC_REGEX)].length > 0; diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index d6b54dc..4f32f12 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -287,8 +287,9 @@ function isEnabled() { const WEIGHT_REGEX = /[([]([^,()[\]:| ]+)(?::(?:\d+(?:\.\d+)?|\.\d+))?[)\]]/g; const POINTY_REGEX = /<[^\s,<](?:[^\t\n\r,<>]*>|[^\t\n\r,> ]*)/g; +const COMPLETED_WILDCARD_REGEX = /__[^\s,_][^\t\n\r,_]*[^\s,_]__[^\s,_]*/g; const NORMAL_TAG_REGEX = /[^\s,|<>]+| Date: Sat, 11 Feb 2023 13:36:39 +0100 Subject: [PATCH 23/26] Extract sanitization / text edit before insertion --- javascript/__globals.js | 1 + javascript/ext_embeddings.js | 12 ++++++++++-- javascript/ext_hypernets.js | 12 ++++++++++-- javascript/ext_loras.js | 12 ++++++++++-- javascript/ext_umi.js | 16 +++++++++++++--- javascript/ext_wildcards.js | 17 +++++++++++++---- javascript/tagAutocomplete.js | 32 ++++++++++++-------------------- 7 files changed, 69 insertions(+), 33 deletions(-) diff --git a/javascript/__globals.js b/javascript/__globals.js index af572be..01ab767 100644 --- a/javascript/__globals.js +++ b/javascript/__globals.js @@ -44,6 +44,7 @@ const QUEUE_AFTER_INSERT = []; const QUEUE_AFTER_SETUP = []; const QUEUE_FILE_LOAD = []; const QUEUE_AFTER_CONFIG_CHANGE = []; +const QUEUE_SANITIZE = []; // List of parsers to try const PARSERS = []; \ No newline at end of file diff --git a/javascript/ext_embeddings.js b/javascript/ext_embeddings.js index 15070b7..409118c 100644 --- a/javascript/ext_embeddings.js +++ b/javascript/ext_embeddings.js @@ -44,7 +44,15 @@ async function load() { } } +function sanitize(tagType, text) { + if (tagType === ResultType.embedding) { + return text.replace(/^.*?: /g, ""); + } + return null; +} + PARSERS.push(new EmbeddingParser(EMB_TRIGGER)); -// Add load function to the queue -QUEUE_FILE_LOAD.push(load); \ No newline at end of file +// Add our utility functions to their respective queues +QUEUE_FILE_LOAD.push(load); +QUEUE_SANITIZE.push(sanitize); \ No newline at end of file diff --git a/javascript/ext_hypernets.js b/javascript/ext_hypernets.js index 9b5dd4c..80787ee 100644 --- a/javascript/ext_hypernets.js +++ b/javascript/ext_hypernets.js @@ -36,7 +36,15 @@ async function load() { } } +function sanitize(tagType, text) { + if (tagType === ResultType.hypernetwork) { + return ``; + } + return null; +} + PARSERS.push(new HypernetParser(HYP_TRIGGER)); -// Add load function to the queue -QUEUE_FILE_LOAD.push(load); \ No newline at end of file +// Add our utility functions to their respective queues +QUEUE_FILE_LOAD.push(load); +QUEUE_SANITIZE.push(sanitize); \ No newline at end of file diff --git a/javascript/ext_loras.js b/javascript/ext_loras.js index a9b7051..3bd67be 100644 --- a/javascript/ext_loras.js +++ b/javascript/ext_loras.js @@ -36,7 +36,15 @@ async function load() { } } +function sanitize(tagType, text) { + if (tagType === ResultType.lora) { + return ``; + } + return null; +} + PARSERS.push(new LoraParser(LORA_TRIGGER)); -// Add load function to the queue -QUEUE_FILE_LOAD.push(load); \ No newline at end of file +// Add our utility functions to their respective queues +QUEUE_FILE_LOAD.push(load); +QUEUE_SANITIZE.push(sanitize); \ No newline at end of file diff --git a/javascript/ext_umi.js b/javascript/ext_umi.js index 9384e97..36a1606 100644 --- a/javascript/ext_umi.js +++ b/javascript/ext_umi.js @@ -223,8 +223,18 @@ async function load() { } } +function sanitize(tagType, text) { + // Replace underscores only if the yaml tag is not using them + if (tagType === ResultType.yamlWildcard && !yamlWildcards.includes(text)) { + return text.replaceAll("_", " "); + } + return null; +} + // Add UMI parser PARSERS.push(new UmiParser(UMI_TRIGGER)); -// Add tag update after insert -QUEUE_AFTER_INSERT.push(updateUmiTags); -QUEUE_FILE_LOAD.push(load); \ No newline at end of file + +// Add our utility functions to their respective queues +QUEUE_FILE_LOAD.push(load); +QUEUE_SANITIZE.push(sanitize); +QUEUE_AFTER_INSERT.push(updateUmiTags); \ No newline at end of file diff --git a/javascript/ext_wildcards.js b/javascript/ext_wildcards.js index b4eae15..a0f7db2 100644 --- a/javascript/ext_wildcards.js +++ b/javascript/ext_wildcards.js @@ -93,6 +93,15 @@ async function load() { } } +function sanitize(tagType, text) { + if (tagType === ResultType.wildcardFile) { + return `__${text}__`; + } else if (tagType === ResultType.wildcardTag) { + return text.replace(/^.*?: /g, ""); + } + return null; +} + function keepOpenIfWildcard(tagType, sanitizedText, newPrompt, textArea) { // If it's a wildcard, we want to keep the results open so the user can select another wildcard if (tagType === ResultType.wildcardFile) { @@ -108,7 +117,7 @@ function keepOpenIfWildcard(tagType, sanitizedText, newPrompt, textArea) { PARSERS.push(new WildcardParser(WC_TRIGGER)); PARSERS.push(new WildcardFileParser(WC_FILE_TRIGGER)); -// Add the keep open function to the queue -QUEUE_AFTER_INSERT.push(keepOpenIfWildcard); -// Add the load function to the queue -QUEUE_FILE_LOAD.push(load); \ No newline at end of file +// Add our utility functions to their respective queues +QUEUE_FILE_LOAD.push(load); +QUEUE_SANITIZE.push(sanitize); +QUEUE_AFTER_INSERT.push(keepOpenIfWildcard); \ No newline at end of file diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 4f32f12..59b2c84 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -299,29 +299,21 @@ async function insertTextAtCursor(textArea, result, tagword) { let cursorPos = textArea.selectionStart; var sanitizedText = text - // Replace differently depending on if it's a tag or wildcard - if (tagType === ResultType.wildcardFile) { - sanitizedText = "__" + text.replace("Wildcards: ", "") + "__"; - } else if (tagType === ResultType.wildcardTag) { - sanitizedText = text.replace(/^.*?: /g, ""); - } else if (tagType === ResultType.yamlWildcard && !yamlWildcards.includes(text)) { - sanitizedText = text.replaceAll("_", " "); // Replace underscores only if the yaml tag is not using them - } else if (tagType === ResultType.embedding) { - sanitizedText = `${text.replace(/^.*?: /g, "")}`; - } else if (tagType === ResultType.hypernetwork) { - sanitizedText = ``; - } else if(tagType === ResultType.lora) { - sanitizedText = ``; + // Run sanitize queue and use first result as sanitized text + sanitizeResults = await processQueueReturn(QUEUE_SANITIZE, null, tagType, text); + + if (sanitizeResults && sanitizeResults.length > 0) { + sanitizedText = sanitizeResults[0]; } else { sanitizedText = CFG.replaceUnderscores ? text.replaceAll("_", " ") : text; - } - if (CFG.escapeParentheses && tagType === ResultType.tag) { - sanitizedText = sanitizedText - .replaceAll("(", "\\(") - .replaceAll(")", "\\)") - .replaceAll("[", "\\[") - .replaceAll("]", "\\]"); + if (CFG.escapeParentheses && tagType === ResultType.tag) { + sanitizedText = sanitizedText + .replaceAll("(", "\\(") + .replaceAll(")", "\\)") + .replaceAll("[", "\\[") + .replaceAll("]", "\\]"); + } } var prompt = textArea.value; From f2c3574da7a851a18ce2846725ddf5a736e44e1b Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sat, 11 Feb 2023 14:13:42 +0100 Subject: [PATCH 24/26] Rework extra file system Now just for adding new custom tags either before or after the rest --- javascript/__globals.js | 1 + javascript/_result.js | 13 +++--- javascript/tagAutocomplete.js | 64 +++++++++++++----------------- scripts/tag_autocomplete_helper.py | 4 +- 4 files changed, 37 insertions(+), 45 deletions(-) diff --git a/javascript/__globals.js b/javascript/__globals.js index 01ab767..64df353 100644 --- a/javascript/__globals.js +++ b/javascript/__globals.js @@ -5,6 +5,7 @@ var tagBasePath = ""; // Tag completion data loaded from files var allTags = []; var translations = new Map(); +var extras = []; // Same for tag-likes var wildcardFiles = []; var wildcardExtFiles = []; diff --git a/javascript/_result.js b/javascript/_result.js index 4799e81..bba990a 100644 --- a/javascript/_result.js +++ b/javascript/_result.js @@ -3,12 +3,13 @@ // Type enum const ResultType = Object.freeze({ "tag": 1, - "embedding": 2, - "wildcardTag": 3, - "wildcardFile": 4, - "yamlWildcard": 5, - "hypernetwork": 6, - "lora": 7 + "extra": 2, + "embedding": 3, + "wildcardTag": 4, + "wildcardFile": 5, + "yamlWildcard": 6, + "hypernetwork": 7, + "lora": 8 }); // Class to hold result data and annotations to make it clearer to use diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 59b2c84..8d74fdb 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -92,42 +92,13 @@ async function loadTags(c) { console.error("Error loading tags file: " + e); return; } - if (c.extra.extraFile && c.extra.extraFile !== "None") { - try { - extras = await loadCSV(`${tagBasePath}/${c.extra.extraFile}`); - if (c.extra.onlyAliasExtraFile) { - // This works purely on index, so it's not very robust. But a lot faster. - for (let i = 0, n = extras.length; i < n; i++) { - if (extras[i][0]) { - let aliasStr = allTags[i][3] || ""; - let optComma = aliasStr.length > 0 ? "," : ""; - allTags[i][3] = aliasStr + optComma + extras[i][0]; - } - } - } else { - extras.forEach(e => { - let hasCount = e[2] && e[3] || (!isNaN(e[2]) && !e[3]); - // Check if a tag in allTags has the same name & category as the extra tag - if (tag = allTags.find(t => t[0] === e[0] && t[1] == e[1])) { - if (hasCount && e[3] || isNaN(e[2])) { // If the extra tag has a translation / alias, add it to the normal tag - let aliasStr = tag[3] || ""; - let optComma = aliasStr.length > 0 ? "," : ""; - let alias = hasCount && e[3] || isNaN(e[2]) ? e[2] : e[3]; - tag[3] = aliasStr + optComma + alias; - } - } else { - let count = hasCount ? e[2] : null; - let aliases = hasCount && e[3] ? e[3] : e[2]; - // If the tag doesn't exist, add it to allTags - let newTag = [e[0], e[1], count, aliases]; - allTags.push(newTag); - } - }); - } - } catch (e) { - console.error("Error loading extra file: " + e); - return; - } + } + if (c.extra.extraFile && c.extra.extraFile !== "None") { + try { + extras = await loadCSV(`${tagBasePath}/${c.extra.extraFile}`); + } catch (e) { + console.error("Error loading extra file: " + e); + return; } } } @@ -191,7 +162,7 @@ async function syncOptions() { // Extra file settings extra: { extraFile: opts["tac_extra.extraFile"], - onlyAliasExtraFile: opts["tac_extra.onlyAliasExtraFile"] + addMode: opts["tac_extra.addMode"] }, // Settings not from tac but still used by the script extraNetworksDefaultMultiplier: opts["extra_networks_default_multiplier"] @@ -671,6 +642,25 @@ async function autocomplete(textArea, prompt, fixedTag = null) { result.aliases = t[3]; results.push(result); }); + + // Add extras + if (CFG.extra.extraFile) { + let extraResults = []; + + extras.filter(fil).forEach(e => { + let result = new AutocompleteResult(e[0].trim(), ResultType.extra) + result.category = e[1] || 0; // If no category is given, use 0 as the default + result.meta = e[2] || "Custom tag"; + result.aliases = e[3] || ""; + extraResults.push(result); + }); + + if (CFG.extra.addMode === "Insert before") { + results = extraResults.concat(results); + } else { + results = results.concat(extraResults); + } + } } // Slice if the user has set a max result count if (!CFG.showAllResults) { diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py index e7a48d6..66c09f3 100644 --- a/scripts/tag_autocomplete_helper.py +++ b/scripts/tag_autocomplete_helper.py @@ -268,7 +268,7 @@ def on_ui_settings(): shared.opts.add_option("tac_translation.oldFormat", shared.OptionInfo(False, "Translation file uses old 3-column translation format instead of the new 2-column one", section=TAC_SECTION)) shared.opts.add_option("tac_translation.searchByTranslation", shared.OptionInfo(True, "Search by translation", section=TAC_SECTION)) # Extra file settings - shared.opts.add_option("tac_extra.extraFile", shared.OptionInfo("None", "Extra filename (do not use e621.csv here!)", gr.Dropdown, lambda: {"choices": csv_files_withnone}, refresh=update_tag_files, section=TAC_SECTION)) - shared.opts.add_option("tac_extra.onlyAliasExtraFile", shared.OptionInfo(False, "Extra file in alias only format", section=TAC_SECTION)) + shared.opts.add_option("tac_extra.extraFile", shared.OptionInfo("None", "Extra filename (for small sets of custom tags)", gr.Dropdown, lambda: {"choices": csv_files_withnone}, refresh=update_tag_files, section=TAC_SECTION)) + shared.opts.add_option("tac_extra.addMode", shared.OptionInfo("Insert before", "Mode to add the extra tags to the main tag list", gr.Dropdown, lambda: {"choices": ["Insert before","Insert after"]}, section=TAC_SECTION)) script_callbacks.on_ui_settings(on_ui_settings) From c16d110de3c921c87e953f4f5f9352ed9144f5aa Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sat, 11 Feb 2023 14:14:48 +0100 Subject: [PATCH 25/26] Add example extra file for the common quality tags --- tags/Extra-quality-tags.csv | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tags/Extra-quality-tags.csv diff --git a/tags/Extra-quality-tags.csv b/tags/Extra-quality-tags.csv new file mode 100644 index 0000000..5461480 --- /dev/null +++ b/tags/Extra-quality-tags.csv @@ -0,0 +1,6 @@ +masterpiece,5,Quality tag, +best_quality,5,Quality tag, +high_quality,5,Quality tag, +normal_quality,5,Quality tag, +low_quality,5,Quality tag, +worst_quality,5,Quality tag, \ No newline at end of file From 37e1c15e6d8ffa03d619367b686ced8f4b09876e Mon Sep 17 00:00:00 2001 From: Dominik Reh Date: Sat, 11 Feb 2023 14:16:31 +0100 Subject: [PATCH 26/26] Make quality tags file the default --- scripts/tag_autocomplete_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py index 66c09f3..97e0d24 100644 --- a/scripts/tag_autocomplete_helper.py +++ b/scripts/tag_autocomplete_helper.py @@ -268,7 +268,7 @@ def on_ui_settings(): shared.opts.add_option("tac_translation.oldFormat", shared.OptionInfo(False, "Translation file uses old 3-column translation format instead of the new 2-column one", section=TAC_SECTION)) shared.opts.add_option("tac_translation.searchByTranslation", shared.OptionInfo(True, "Search by translation", section=TAC_SECTION)) # Extra file settings - shared.opts.add_option("tac_extra.extraFile", shared.OptionInfo("None", "Extra filename (for small sets of custom tags)", gr.Dropdown, lambda: {"choices": csv_files_withnone}, refresh=update_tag_files, section=TAC_SECTION)) + shared.opts.add_option("tac_extra.extraFile", shared.OptionInfo("extra-quality-tags.csv", "Extra filename (for small sets of custom tags)", gr.Dropdown, lambda: {"choices": csv_files_withnone}, refresh=update_tag_files, section=TAC_SECTION)) shared.opts.add_option("tac_extra.addMode", shared.OptionInfo("Insert before", "Mode to add the extra tags to the main tag list", gr.Dropdown, lambda: {"choices": ["Insert before","Insert after"]}, section=TAC_SECTION)) script_callbacks.on_ui_settings(on_ui_settings)