ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-03-09 05:20:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

llama : rename jina tokenizers to v2 (#7249)

Browse Source 
* refactor: rename jina tokenizers to v2

* refactor: keep refactoring non-breaking
This commit is contained in:
Joan Fontanals
2024-05-13 10:35:14 +02:00
committed by GitHub
parent 97684629fa
commit bf009f1d45
3 changed files with 9 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
llama.cpp
View File

@@ -4424,7 +4424,9 @@ static void llm_load_vocab(
} else if (
tokenizer_pre == "gpt-2" ||
tokenizer_pre == "jina-es" ||
tokenizer_pre == "jina-de") {
tokenizer_pre == "jina-de" ||
tokenizer_pre == "jina-v2-es" ||
tokenizer_pre == "jina-v2-de") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;
} else if (
tokenizer_pre == "refact") {