From 83461e2f545e8034b1c8dfdacd952c9124662565 Mon Sep 17 00:00:00 2001 From: DominikDoom Date: Sat, 22 Apr 2023 21:07:08 +0200 Subject: [PATCH 01/10] Ruby Text live translation feature (WIP) --- javascript/tagAutocomplete.js | 79 +++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 5240eea..158304a 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -8,6 +8,9 @@ "--meta-text-color": ["#6b6f7b", "#a2a9b4"], "--embedding-v1-color": ["lightsteelblue", "#2b5797"], "--embedding-v2-color": ["skyblue", "#2d89ef"], + "--live-translation-color-1": ["lightskyblue", "#2d89ef"], + "--live-translation-color-2": ["palegoldenrod", "#eb5700"], + "--live-translation-color-3": ["darkseagreen", "darkgreen"], } const browserVars = { "--results-overflow-y": { @@ -78,6 +81,26 @@ const autocompleteCSS = ` .acListItem.acEmbeddingV2 { color: var(--embedding-v2-color); } + .acRuby { + margin-top: 0.5rem; + padding: var(--input-padding); + color: #888; + } + .acRuby > :nth-child(3n+1) { + color: var(--live-translation-color-1); + } + .acRuby > :nth-child(3n+2) { + color: var(--live-translation-color-2); + } + .acRuby > :nth-child(3n+3) { + color: var(--live-translation-color-3); + } + .acRuby > ruby > rt { + overflow: hidden; + padding: 0px 5px; + text-align: center; + font-size: 0.8em; + } `; async function loadTags(c) { @@ -217,7 +240,7 @@ function createResultsDiv(textArea) { let typeClass = textAreaId.replaceAll(".", " "); resultsDiv.style.maxHeight = `${CFG.maxResults * 50}px`; - resultsDiv.setAttribute("class", `autocompleteResults ${typeClass} notranslate`); + resultsDiv.setAttribute("class", `autocompleteResults${typeClass} notranslate`); resultsDiv.setAttribute("translate", "no"); resultsList.setAttribute("class", "autocompleteResultsList"); resultsDiv.appendChild(resultsList); @@ -286,6 +309,7 @@ const WEIGHT_REGEX = /[([]([^()[\]:|]+)(?::(?:\d+(?:\.\d+)?|\.\d+))?[)\]]/g; const POINTY_REGEX = /<[^\s,<](?:[^\t\n\r,<>]*>|[^\t\n\r,> ]*)/g; const COMPLETED_WILDCARD_REGEX = /__[^\s,_][^\t\n\r,_]*[^\s,_]__[^\s,_]*/g; const NORMAL_TAG_REGEX = /[^\s,|<>)\]]+| { + tag = tag.trim(); + // Cut off opening bracket for weighted tags + if (tag.startsWith("(")) { + tag = tag.substring(1); + // Cut off closing bracket if left over from a single word weighted tag + if (tag.endsWith(")")) + tag = tag.substring(0, tag.length - 1); + } + + let unsanitizedTag = tag + .replaceAll(" ", "_") + .replaceAll("\\(", "(") + .replaceAll("\\)", ")"); + + const translation = translations?.get(tag) || translations?.get(unsanitizedTag); + + let escapedTag = tag; + if (tag.endsWith("\\)")) { + escapedTag = escapedTag.substring(0, escapedTag.length - 1); + } + escapedTag = escapeRegExp(escapedTag); + + if (translation) + ruby.innerHTML = ruby.innerHTML.replaceAll(new RegExp(`${escapedTag}(?:\\)|\\b)`, "g"), `${translation}${tag}`); + }); +} + async function autocomplete(textArea, prompt, fixedTag = null) { // Return if the function is deactivated in the UI if (!isEnabled()) return; @@ -875,7 +943,10 @@ async function setup() { hideResults(area); // Add autocomplete event listener - area.addEventListener('input', debounce(() => autocomplete(area, area.value), CFG.delayTime)); + area.addEventListener('input', debounce(() => { + autocomplete(area, area.value); + updateRuby(area, area.value); + }, CFG.delayTime)); // Add focusout event listener area.addEventListener('focusout', debounce(() => hideResults(area), 400)); // Add up and down arrow event listener @@ -903,10 +974,10 @@ async function setup() { let css = autocompleteCSS; // Replace vars with actual values (can't use actual css vars because of the way we inject the css) Object.keys(styleColors).forEach((key) => { - css = css.replace(`var(${key})`, styleColors[key][mode]); + css = css.replaceAll(`var(${key})`, styleColors[key][mode]); }) Object.keys(browserVars).forEach((key) => { - css = css.replace(`var(${key})`, browserVars[key][browser]); + css = css.replaceAll(`var(${key})`, browserVars[key][browser]); }) if (acStyle.styleSheet) { From b18823e88f383cafd28b332f93a5a070ef855dc0 Mon Sep 17 00:00:00 2001 From: DominikDoom Date: Sat, 22 Apr 2023 22:33:48 +0200 Subject: [PATCH 02/10] Sliding window search, fix double replacement --- javascript/_utils.js | 8 ++++++++ javascript/tagAutocomplete.js | 37 +++++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/javascript/_utils.js b/javascript/_utils.js index 863b4d3..0804f5e 100644 --- a/javascript/_utils.js +++ b/javascript/_utils.js @@ -89,6 +89,14 @@ function difference(a, b) { )].reduce((acc, [v, count]) => acc.concat(Array(Math.abs(count)).fill(v)), []); } +// Sliding window function to get possible combination groups of an array +function toWindows(inputArray, size) { + return Array.from( + {length: inputArray.length - (size - 1)}, //get the appropriate length + (_, index) => inputArray.slice(index, index+size) //create the windows + ) + } + function escapeRegExp(string) { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string } diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 158304a..9f3b545 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -611,8 +611,41 @@ async function updateRuby(textArea, prompt) { } escapedTag = escapeRegExp(escapedTag); - if (translation) - ruby.innerHTML = ruby.innerHTML.replaceAll(new RegExp(`${escapedTag}(?:\\)|\\b)`, "g"), `${translation}${tag}`); + if (translation) { + ruby.innerHTML = ruby.innerHTML.replaceAll(new RegExp(`(?)${escapedTag}(?:\\)|\\b)(?!)`, "g"), `${tag}${translation}`); + } else { + // No direct match, but we can try to find matches in 2 or 1 word windows + let subTags = tag.split(" "); + // Return if there is only one word + if (subTags.length === 1) return; + + const translateWindows = function (windows) { + windows.forEach(window => { + let windowTag = window.join(" "); + let unsanitizedWindowTag = windowTag + .replaceAll(" ", "_") + .replaceAll("\\(", "(") + .replaceAll("\\)", ")"); + + let translation = translations?.get(windowTag) || translations?.get(unsanitizedWindowTag); + + let escapedWindowTag = windowTag; + if (windowTag.endsWith("\\)")) { + escapedWindowTag = escapedWindowTag.substring(0, escapedWindowTag.length - 1); + } + escapedWindowTag = escapeRegExp(escapedWindowTag); + + if (translation) { + ruby.innerHTML = ruby.innerHTML.replaceAll(new RegExp(`(?)${escapedWindowTag}(?:\\)|\\b)(?!)`, "g"), `${windowTag}${translation}`); + subTags = subTags.filter(tag => !window.includes(tag)); + } + }); + } + + // Get sliding windows of 2 words and each word individually + translateWindows(toWindows(subTags, 2), subTags); + translateWindows(toWindows(subTags, 1), subTags); + } }); } From 85db4a61df72e154f0d4b15cb517deca74a36291 Mon Sep 17 00:00:00 2001 From: DominikDoom Date: Sun, 23 Apr 2023 13:28:39 +0200 Subject: [PATCH 03/10] Adjust styling for bigger translation --- javascript/tagAutocomplete.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 9f3b545..c85637f 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -86,6 +86,12 @@ const autocompleteCSS = ` padding: var(--input-padding); color: #888; } + .acRuby > ruby { + display: inline-flex; + flex-direction: column-reverse; + font-size: 0.8rem; + vertical-align: text-bottom; + } .acRuby > :nth-child(3n+1) { color: var(--live-translation-color-1); } @@ -96,10 +102,11 @@ const autocompleteCSS = ` color: var(--live-translation-color-3); } .acRuby > ruby > rt { - overflow: hidden; - padding: 0px 5px; - text-align: center; - font-size: 0.8em; + line-height: 1rem; + padding: 0px 5px 0px 0px; + text-align: left; + font-size: 1rem; + color: var(--block-title-text-color); } `; From 38fd2523e6d62bb5e30501700ef34c1bd5383b97 Mon Sep 17 00:00:00 2001 From: DominikDoom Date: Sun, 23 Apr 2023 17:05:38 +0200 Subject: [PATCH 04/10] Fix issue with self-containing tags, add auto select on click --- javascript/tagAutocomplete.js | 110 ++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 37 deletions(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index c85637f..59bfaa9 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -85,12 +85,18 @@ const autocompleteCSS = ` margin-top: 0.5rem; padding: var(--input-padding); color: #888; + font-size: 0.8rem; + user-select: none; } .acRuby > ruby { display: inline-flex; flex-direction: column-reverse; - font-size: 0.8rem; - vertical-align: text-bottom; + vertical-align: bottom; + cursor: pointer; + } + .acRuby > ruby::hover { + text-decoration: underline; + text-shadow: 0 0 10px var(--live-translation-color-1); } .acRuby > :nth-child(3n+1) { color: var(--live-translation-color-1); @@ -579,7 +585,7 @@ function updateSelectionStyle(textArea, newIndex, oldIndex) { } } -async function updateRuby(textArea, prompt) { +function updateRuby(textArea, prompt) { let ruby = gradioApp().querySelector('.acRuby' + getTextAreaIdentifier(textArea)); if (!ruby) { let textAreaId = getTextAreaIdentifier(textArea); @@ -594,8 +600,10 @@ async function updateRuby(textArea, prompt) { let rubyTags = prompt.match(RUBY_TAG_REGEX); if (!rubyTags) return; + //rubyTags.sort((a, b) => b.length - a.length); rubyTags = new Set(rubyTags); - rubyTags.forEach(tag => { + + const prepareTag = function (tag) { tag = tag.trim(); // Cut off opening bracket for weighted tags if (tag.startsWith("(")) { @@ -610,7 +618,7 @@ async function updateRuby(textArea, prompt) { .replaceAll("\\(", "(") .replaceAll("\\)", ")"); - const translation = translations?.get(tag) || translations?.get(unsanitizedTag); + const translation = translations?.get(unsanitizedTag) || translations?.get(tag) let escapedTag = tag; if (tag.endsWith("\\)")) { @@ -618,42 +626,70 @@ async function updateRuby(textArea, prompt) { } escapedTag = escapeRegExp(escapedTag); - if (translation) { - ruby.innerHTML = ruby.innerHTML.replaceAll(new RegExp(`(?)${escapedTag}(?:\\)|\\b)(?!)`, "g"), `${tag}${translation}`); - } else { - // No direct match, but we can try to find matches in 2 or 1 word windows - let subTags = tag.split(" "); - // Return if there is only one word - if (subTags.length === 1) return; + return { tag, escapedTag, translation }; + } - const translateWindows = function (windows) { - windows.forEach(window => { - let windowTag = window.join(" "); - let unsanitizedWindowTag = windowTag - .replaceAll(" ", "_") - .replaceAll("\\(", "(") - .replaceAll("\\)", ")"); - - let translation = translations?.get(windowTag) || translations?.get(unsanitizedWindowTag); - - let escapedWindowTag = windowTag; - if (windowTag.endsWith("\\)")) { - escapedWindowTag = escapedWindowTag.substring(0, escapedWindowTag.length - 1); - } - escapedWindowTag = escapeRegExp(escapedWindowTag); - - if (translation) { - ruby.innerHTML = ruby.innerHTML.replaceAll(new RegExp(`(?)${escapedWindowTag}(?:\\)|\\b)(?!)`, "g"), `${windowTag}${translation}`); - subTags = subTags.filter(tag => !window.includes(tag)); - } - }); - } + const replaceOccurences = (text, tuple) => { + let { tag, escapedTag, translation } = tuple; + let searchRegex = new RegExp(`(?)(?:\\(|\\b)${escapedTag}(?:\\)|\\b)(?!)`, "g"); + return text.replaceAll(searchRegex, `${tag}${translation}`); + } - // Get sliding windows of 2 words and each word individually - translateWindows(toWindows(subTags, 2), subTags); - translateWindows(toWindows(subTags, 1), subTags); + let html = ruby.innerHTML; + + // First do n-gram search to cover all 3-1 word combinations + rubyTags.forEach(tag => { + let subTags = tag.split(" ").filter(x => x.trim().length > 0); + // Return if there is only one word + //if (subTags.length === 1) return; + + const translateWindows = (windows) => { + windows.forEach(window => { + let combinedTag = window.join(" "); + let tuple = prepareTag(combinedTag); + if (tuple.translation) { + html = replaceOccurences(html, tuple); + } + }); + } + + // Perform n-gram sliding window search + translateWindows(toWindows(subTags, 3)); + // Looks ugly, but is needed to ensure the DOM updated before we translate the next window + requestAnimationFrame(() => translateWindows(toWindows(subTags, 2))); + requestAnimationFrame(() => translateWindows(toWindows(subTags, 1))); + }); + + // Then try direct matches for all that are still untranslated + let preparedTags = [...rubyTags].map(prepareTag); + preparedTags.forEach(tuple => { + if (tuple.translation) { + html = replaceOccurences(html, tuple); } }); + + ruby.innerHTML = html; + + requestAnimationFrame(() => requestAnimationFrame(() => { + // Add listeners for auto selection + const childNodes = [...ruby.childNodes]; + [...ruby.children].forEach(child => { + const textBefore = childNodes.slice(0, childNodes.indexOf(child)).map(x => x.childNodes[0]?.textContent || x.textContent).join("") + child.onclick = () => rubyTagClicked(child, textBefore, prompt, textArea); + }); + })) +} + +function rubyTagClicked(node, textBefore, prompt, textArea) { + let selectionText = node.childNodes[0].textContent; + + // Find start and end position of the tag in the prompt + let startPos = prompt.indexOf(textBefore) + textBefore.length; + let endPos = startPos + selectionText.length; + + // Select in text area + textArea.focus(); + textArea.setSelectionRange(startPos, endPos); } async function autocomplete(textArea, prompt, fixedTag = null) { From 84b6a0394e63ae95057f4fdb70245b22548c66f0 Mon Sep 17 00:00:00 2001 From: DominikDoom Date: Sun, 23 Apr 2023 19:58:49 +0200 Subject: [PATCH 05/10] Fix more bugs related to replacing Also some involving parentheses --- javascript/_utils.js | 10 ++-- javascript/tagAutocomplete.js | 110 +++++++++++++++++++--------------- 2 files changed, 67 insertions(+), 53 deletions(-) diff --git a/javascript/_utils.js b/javascript/_utils.js index 0804f5e..476e1d1 100644 --- a/javascript/_utils.js +++ b/javascript/_utils.js @@ -90,12 +90,12 @@ function difference(a, b) { } // Sliding window function to get possible combination groups of an array -function toWindows(inputArray, size) { +function toNgrams(inputArray, size) { return Array.from( - {length: inputArray.length - (size - 1)}, //get the appropriate length - (_, index) => inputArray.slice(index, index+size) //create the windows - ) - } + { length: inputArray.length - (size - 1) }, //get the appropriate length + (_, index) => inputArray.slice(index, index + size) //create the windows + ); +} function escapeRegExp(string) { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 59bfaa9..f59a673 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -82,7 +82,6 @@ const autocompleteCSS = ` color: var(--embedding-v2-color); } .acRuby { - margin-top: 0.5rem; padding: var(--input-padding); color: #888; font-size: 0.8rem; @@ -91,6 +90,7 @@ const autocompleteCSS = ` .acRuby > ruby { display: inline-flex; flex-direction: column-reverse; + margin-top: 0.5rem; vertical-align: bottom; cursor: pointer; } @@ -594,23 +594,34 @@ function updateRuby(textArea, prompt) { ruby.setAttribute("class", `acRuby${typeClass} notranslate`); textArea.parentNode.appendChild(ruby); } - + ruby.innerText = prompt; let rubyTags = prompt.match(RUBY_TAG_REGEX); if (!rubyTags) return; - //rubyTags.sort((a, b) => b.length - a.length); + rubyTags.sort((a, b) => b.length - a.length); rubyTags = new Set(rubyTags); - const prepareTag = function (tag) { + const prepareTag = (tag) => { tag = tag.trim(); // Cut off opening bracket for weighted tags if (tag.startsWith("(")) { - tag = tag.substring(1); + // Find index of last opening bracket + let openIndex = 0; + while (tag.charAt(openIndex) === "(") { + openIndex++; + tag = tag.substring(openIndex); + } + // Cut off closing bracket if left over from a single word weighted tag - if (tag.endsWith(")")) - tag = tag.substring(0, tag.length - 1); + if (tag.endsWith(")")) { + let closeIndex = tag.length - 1; + while (tag.charAt(closeIndex) === ")") { + closeIndex--; + tag = tag.substring(0, closeIndex + 1); + } + } } let unsanitizedTag = tag @@ -618,7 +629,7 @@ function updateRuby(textArea, prompt) { .replaceAll("\\(", "(") .replaceAll("\\)", ")"); - const translation = translations?.get(unsanitizedTag) || translations?.get(tag) + const translation = translations?.get(tag) || translations?.get(unsanitizedTag); let escapedTag = tag; if (tag.endsWith("\\)")) { @@ -631,53 +642,56 @@ function updateRuby(textArea, prompt) { const replaceOccurences = (text, tuple) => { let { tag, escapedTag, translation } = tuple; - let searchRegex = new RegExp(`(?)(?:\\(|\\b)${escapedTag}(?:\\)|\\b)(?!)`, "g"); - return text.replaceAll(searchRegex, `${tag}${translation}`); + let searchRegex = new RegExp(`(?)(?:\\b)${escapedTag}(?:\\)|\\b)(?!)`, "g"); + return text.replaceAll(searchRegex, `${escapeHTML(tag)}${translation}`); } - let html = ruby.innerHTML; + let html = escapeHTML(prompt); - // First do n-gram search to cover all 3-1 word combinations - rubyTags.forEach(tag => { - let subTags = tag.split(" ").filter(x => x.trim().length > 0); - // Return if there is only one word - //if (subTags.length === 1) return; - - const translateWindows = (windows) => { - windows.forEach(window => { - let combinedTag = window.join(" "); - let tuple = prepareTag(combinedTag); - if (tuple.translation) { - html = replaceOccurences(html, tuple); - } - }); - } - - // Perform n-gram sliding window search - translateWindows(toWindows(subTags, 3)); - // Looks ugly, but is needed to ensure the DOM updated before we translate the next window - requestAnimationFrame(() => translateWindows(toWindows(subTags, 2))); - requestAnimationFrame(() => translateWindows(toWindows(subTags, 1))); - }); - - // Then try direct matches for all that are still untranslated - let preparedTags = [...rubyTags].map(prepareTag); - preparedTags.forEach(tuple => { + // First try to find direct matches + [...rubyTags].forEach(tag => { + let tuple = prepareTag(tag); + if (tuple.translation) { html = replaceOccurences(html, tuple); + } else { + let subTags = tuple.tag.split(" ").filter(x => x.trim().length > 0); + // Return if there is only one word + if (subTags.length === 1) return; + + let subHtml = tag; + + let translateNgram = (windows) => { + windows.forEach(window => { + let combinedTag = window.join(" "); + let subTuple = prepareTag(combinedTag); + + if (subTuple.tag.length <= 2) return; + + if (subTuple.translation) { + subHtml = replaceOccurences(subHtml, subTuple); + } + }); + } + + // Perform n-gram sliding window search + translateNgram(toNgrams(subTags, 3)); + translateNgram(toNgrams(subTags, 2)); + translateNgram(toNgrams(subTags, 1)); + + let searchRegex = new RegExp(`(?)(?:\\(|\\b)+${escapeRegExp(tuple.tag)}(?:\\)|\\b)+(?!)`, "g"); + html = html.replaceAll(searchRegex, subHtml); } }); ruby.innerHTML = html; - requestAnimationFrame(() => requestAnimationFrame(() => { - // Add listeners for auto selection - const childNodes = [...ruby.childNodes]; - [...ruby.children].forEach(child => { - const textBefore = childNodes.slice(0, childNodes.indexOf(child)).map(x => x.childNodes[0]?.textContent || x.textContent).join("") - child.onclick = () => rubyTagClicked(child, textBefore, prompt, textArea); - }); - })) + // Add listeners for auto selection + const childNodes = [...ruby.childNodes]; + [...ruby.children].forEach(child => { + const textBefore = childNodes.slice(0, childNodes.indexOf(child)).map(x => x.childNodes[0]?.textContent || x.textContent).join("") + child.onclick = () => rubyTagClicked(child, textBefore, prompt, textArea); + }); } function rubyTagClicked(node, textBefore, prompt, textArea) { @@ -1019,10 +1033,10 @@ async function setup() { hideResults(area); // Add autocomplete event listener - area.addEventListener('input', debounce(() => { - autocomplete(area, area.value); + area.addEventListener('input', () => { + debounce(autocomplete(area, area.value), CFG.delayTime); updateRuby(area, area.value); - }, CFG.delayTime)); + }); // Add focusout event listener area.addEventListener('focusout', debounce(() => hideResults(area), 400)); // Add up and down arrow event listener From 3169420fd3cb4c1a2acfb008982cbc5a2213dcf1 Mon Sep 17 00:00:00 2001 From: DominikDoom Date: Tue, 25 Apr 2023 13:37:52 +0200 Subject: [PATCH 06/10] Fix parentheses parsing --- javascript/tagAutocomplete.js | 42 +++++++++-------------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index f59a673..69aab03 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -322,7 +322,7 @@ const WEIGHT_REGEX = /[([]([^()[\]:|]+)(?::(?:\d+(?:\.\d+)?|\.\d+))?[)\]]/g; const POINTY_REGEX = /<[^\s,<](?:[^\t\n\r,<>]*>|[^\t\n\r,> ]*)/g; const COMPLETED_WILDCARD_REGEX = /__[^\s,_][^\t\n\r,_]*[^\s,_]__[^\s,_]*/g; const NORMAL_TAG_REGEX = /[^\s,|<>)\]]+|?/g; const TAG_REGEX = new RegExp(`${POINTY_REGEX.source}|${COMPLETED_WILDCARD_REGEX.source}|${NORMAL_TAG_REGEX.source}`, "g"); // On click, insert the tag into the prompt textbox with respect to the cursor position @@ -596,33 +596,17 @@ function updateRuby(textArea, prompt) { } ruby.innerText = prompt; - - let rubyTags = prompt.match(RUBY_TAG_REGEX); + + let bracketEscapedPrompt = prompt.replaceAll("\\(", "$").replaceAll("\\)", "%"); + + let rubyTags = bracketEscapedPrompt.match(RUBY_TAG_REGEX); if (!rubyTags) return; rubyTags.sort((a, b) => b.length - a.length); rubyTags = new Set(rubyTags); const prepareTag = (tag) => { - tag = tag.trim(); - // Cut off opening bracket for weighted tags - if (tag.startsWith("(")) { - // Find index of last opening bracket - let openIndex = 0; - while (tag.charAt(openIndex) === "(") { - openIndex++; - tag = tag.substring(openIndex); - } - - // Cut off closing bracket if left over from a single word weighted tag - if (tag.endsWith(")")) { - let closeIndex = tag.length - 1; - while (tag.charAt(closeIndex) === ")") { - closeIndex--; - tag = tag.substring(0, closeIndex + 1); - } - } - } + tag = tag.replaceAll("$", "\\(").replaceAll("%", "\\)"); let unsanitizedTag = tag .replaceAll(" ", "_") @@ -631,18 +615,13 @@ function updateRuby(textArea, prompt) { const translation = translations?.get(tag) || translations?.get(unsanitizedTag); - let escapedTag = tag; - if (tag.endsWith("\\)")) { - escapedTag = escapedTag.substring(0, escapedTag.length - 1); - } - escapedTag = escapeRegExp(escapedTag); - + let escapedTag = escapeRegExp(tag); return { tag, escapedTag, translation }; } const replaceOccurences = (text, tuple) => { let { tag, escapedTag, translation } = tuple; - let searchRegex = new RegExp(`(?)(?:\\b)${escapedTag}(?:\\)|\\b)(?!)`, "g"); + let searchRegex = new RegExp(`(?)(?:\\b)${escapedTag}(?!)`, "g"); return text.replaceAll(searchRegex, `${escapeHTML(tag)}${translation}`); } @@ -659,7 +638,7 @@ function updateRuby(textArea, prompt) { // Return if there is only one word if (subTags.length === 1) return; - let subHtml = tag; + let subHtml = tag.replaceAll("$", "\\(").replaceAll("%", "\\)"); let translateNgram = (windows) => { windows.forEach(window => { @@ -679,7 +658,8 @@ function updateRuby(textArea, prompt) { translateNgram(toNgrams(subTags, 2)); translateNgram(toNgrams(subTags, 1)); - let searchRegex = new RegExp(`(?)(?:\\(|\\b)+${escapeRegExp(tuple.tag)}(?:\\)|\\b)+(?!)`, "g"); + let escapedTag = escapeRegExp(tuple.tag.replaceAll("$", "(").replaceAll("%", ")")); + let searchRegex = new RegExp(`(?)(?:\\b)${escapedTag}(?!)`, "g"); html = html.replaceAll(searchRegex, subHtml); } }); From 91fb1cba385f1122fd65999c71679c74d480050d Mon Sep 17 00:00:00 2001 From: DominikDoom Date: Tue, 25 Apr 2023 14:41:29 +0200 Subject: [PATCH 07/10] Fix replacer matching partial words Now checks for end of word / string or tag boundary correctly --- javascript/tagAutocomplete.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 69aab03..5ad9f90 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -621,7 +621,7 @@ function updateRuby(textArea, prompt) { const replaceOccurences = (text, tuple) => { let { tag, escapedTag, translation } = tuple; - let searchRegex = new RegExp(`(?)(?:\\b)${escapedTag}(?!)`, "g"); + let searchRegex = new RegExp(`(?)(?:\\b)${escapedTag}(?:\\b|$|(?=[,| \\t\\n\\r]))(?!)`, "g"); return text.replaceAll(searchRegex, `${escapeHTML(tag)}${translation}`); } @@ -658,8 +658,9 @@ function updateRuby(textArea, prompt) { translateNgram(toNgrams(subTags, 2)); translateNgram(toNgrams(subTags, 1)); - let escapedTag = escapeRegExp(tuple.tag.replaceAll("$", "(").replaceAll("%", ")")); - let searchRegex = new RegExp(`(?)(?:\\b)${escapedTag}(?!)`, "g"); + let escapedTag = escapeRegExp(tuple.tag); + + let searchRegex = new RegExp(`(?)(?:\\b)${escapedTag}(?:\\b|$|(?=[,| \\t\\n\\r]))(?!)`, "g"); html = html.replaceAll(searchRegex, subHtml); } }); From d4941c7b73d390e37accfc01ed3f0d6449a11d78 Mon Sep 17 00:00:00 2001 From: DominikDoom Date: Tue, 25 Apr 2023 14:52:18 +0200 Subject: [PATCH 08/10] Fix rt color --- javascript/tagAutocomplete.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 5ad9f90..1051a87 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -8,6 +8,7 @@ "--meta-text-color": ["#6b6f7b", "#a2a9b4"], "--embedding-v1-color": ["lightsteelblue", "#2b5797"], "--embedding-v2-color": ["skyblue", "#2d89ef"], + "--live-translation-rt": ["whitesmoke", "#222"], "--live-translation-color-1": ["lightskyblue", "#2d89ef"], "--live-translation-color-2": ["palegoldenrod", "#eb5700"], "--live-translation-color-3": ["darkseagreen", "darkgreen"], @@ -112,7 +113,7 @@ const autocompleteCSS = ` padding: 0px 5px 0px 0px; text-align: left; font-size: 1rem; - color: var(--block-title-text-color); + color: var(--live-translation-rt); } `; From 94365630c7e683ef6427ff44ee5d9fdde7d49344 Mon Sep 17 00:00:00 2001 From: DominikDoom Date: Tue, 25 Apr 2023 15:03:04 +0200 Subject: [PATCH 09/10] Fix end-of-tag detection for brackets before a weight modifier --- javascript/tagAutocomplete.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index 1051a87..d88fad2 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -622,7 +622,7 @@ function updateRuby(textArea, prompt) { const replaceOccurences = (text, tuple) => { let { tag, escapedTag, translation } = tuple; - let searchRegex = new RegExp(`(?)(?:\\b)${escapedTag}(?:\\b|$|(?=[,| \\t\\n\\r]))(?!)`, "g"); + let searchRegex = new RegExp(`(?)(?:\\b)${escapedTag}(?:\\b|$|(?=[,|: \\t\\n\\r]))(?!)`, "g"); return text.replaceAll(searchRegex, `${escapeHTML(tag)}${translation}`); } @@ -661,7 +661,7 @@ function updateRuby(textArea, prompt) { let escapedTag = escapeRegExp(tuple.tag); - let searchRegex = new RegExp(`(?)(?:\\b)${escapedTag}(?:\\b|$|(?=[,| \\t\\n\\r]))(?!)`, "g"); + let searchRegex = new RegExp(`(?)(?:\\b)${escapedTag}(?:\\b|$|(?=[,|: \\t\\n\\r]))(?!)`, "g"); html = html.replaceAll(searchRegex, subHtml); } }); From 0e177d094580ce7a113dd5bfd4ae7fcc27dacf7f Mon Sep 17 00:00:00 2001 From: DominikDoom Date: Tue, 2 May 2023 17:59:36 +0200 Subject: [PATCH 10/10] Add option for live preview --- javascript/tagAutocomplete.js | 10 ++++++++++ scripts/tag_autocomplete_helper.py | 1 + 2 files changed, 11 insertions(+) diff --git a/javascript/tagAutocomplete.js b/javascript/tagAutocomplete.js index d88fad2..ed27599 100644 --- a/javascript/tagAutocomplete.js +++ b/javascript/tagAutocomplete.js @@ -202,6 +202,7 @@ async function syncOptions() { translationFile: opts["tac_translation.translationFile"], oldFormat: opts["tac_translation.oldFormat"], searchByTranslation: opts["tac_translation.searchByTranslation"], + liveTranslation: opts["tac_translation.liveTranslation"], }, // Extra file settings extra: { @@ -238,6 +239,13 @@ async function syncOptions() { }); } + // Remove ruby div if live preview was disabled + if (newCFG.translation.liveTranslation === false) { + [...gradioApp().querySelectorAll('.acRuby')].forEach(r => { + r.remove(); + }); + } + // Apply changes CFG = newCFG; @@ -587,6 +595,8 @@ function updateSelectionStyle(textArea, newIndex, oldIndex) { } function updateRuby(textArea, prompt) { + if (!CFG.translation.liveTranslation) return; + let ruby = gradioApp().querySelector('.acRuby' + getTextAreaIdentifier(textArea)); if (!ruby) { let textAreaId = getTextAreaIdentifier(textArea); diff --git a/scripts/tag_autocomplete_helper.py b/scripts/tag_autocomplete_helper.py index b370c42..fc1dd8c 100644 --- a/scripts/tag_autocomplete_helper.py +++ b/scripts/tag_autocomplete_helper.py @@ -298,6 +298,7 @@ def on_ui_settings(): shared.opts.add_option("tac_translation.translationFile", shared.OptionInfo("None", "Translation filename", gr.Dropdown, lambda: {"choices": csv_files_withnone}, refresh=update_tag_files, section=TAC_SECTION)) shared.opts.add_option("tac_translation.oldFormat", shared.OptionInfo(False, "Translation file uses old 3-column translation format instead of the new 2-column one", section=TAC_SECTION)) shared.opts.add_option("tac_translation.searchByTranslation", shared.OptionInfo(True, "Search by translation", section=TAC_SECTION)) + shared.opts.add_option("tac_translation.liveTranslation", shared.OptionInfo(False, "Show live tag translation below prompt", section=TAC_SECTION)) # Extra file settings shared.opts.add_option("tac_extra.extraFile", shared.OptionInfo("extra-quality-tags.csv", "Extra filename (for small sets of custom tags)", gr.Dropdown, lambda: {"choices": csv_files_withnone}, refresh=update_tag_files, section=TAC_SECTION)) shared.opts.add_option("tac_extra.addMode", shared.OptionInfo("Insert before", "Mode to add the extra tags to the main tag list", gr.Dropdown, lambda: {"choices": ["Insert before","Insert after"]}, section=TAC_SECTION))