Extract sanitization / text edit before insertion

This commit is contained in:
Dominik Reh
2023-02-11 13:36:39 +01:00
parent b540400110
commit b4fe4f717a
7 changed files with 69 additions and 33 deletions

View File

@@ -44,7 +44,15 @@ async function load() {
}
}
function sanitize(tagType, text) {
if (tagType === ResultType.embedding) {
return text.replace(/^.*?: /g, "");
}
return null;
}
PARSERS.push(new EmbeddingParser(EMB_TRIGGER));
// Add load function to the queue
QUEUE_FILE_LOAD.push(load);
// Add our utility functions to their respective queues
QUEUE_FILE_LOAD.push(load);
QUEUE_SANITIZE.push(sanitize);