ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-02-05 14:00:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Special handling of Seed Coder FIM tokens (#585)

Browse Source 
* Special handling of Seed Coder FIM tokens

* vocab: Add Seed Coder pretokenizer

* Formatting fix

* Update llama.h
This commit is contained in:
Fizz~
2025-07-06 06:13:55 -04:00
committed by GitHub
parent 49d4d2630a
commit 27ff5bf57e
5 changed files with 27 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/llama.cpp
View File

@@ -6302,6 +6302,10 @@ static void llm_load_vocab(
tokenizer_pre == "bailingmoe") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_BAILINGMOE;
vocab.tokenizer_clean_spaces = false;
} else if (
tokenizer_pre == "seed-coder") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SEED_CODER;
vocab.tokenizer_clean_spaces = false;
} else {
throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
}