ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-01-26 17:20:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix test build error

Browse Source
This commit is contained in:
firecoperana
2026-01-17 10:04:42 -06:00
parent 2403d6ee02
commit 02aa65009b
2 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tests/test-tokenizer-1-bpe.cpp
View File

@@ -78,7 +78,7 @@ int main(int argc, char **argv) {
const int n_vocab = llama_n_vocab(model);
for (int i = 0; i < n_vocab; ++i) {
std::string str = llama_detokenize(ctx, std::vector<int>(1, i));
std::string str = common_token_to_piece(ctx, std::vector<int>(1, i));
try {
auto cps = unicode_cpts_from_utf8(str);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false, true);
@@ -94,7 +94,7 @@ int main(int argc, char **argv) {
fprintf(stderr, "]\n");
return 2;
}
std::string check = llama_detokenize(ctx, tokens);
std::string check = common_token_to_piece(ctx, tokens);
if (check != str) {
fprintf(stderr, "%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n",
__func__, i, str.c_str(), str.length(), check.c_str(), check.length());
@@ -124,7 +124,7 @@ int main(int argc, char **argv) {
std::string str = unicode_cpt_to_utf8(cp);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize(ctx, tokens);
std::string check = common_token_to_piece(ctx, tokens);
if (cp != 9601 && str != check) {
fprintf(stderr, "error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
cp, check.c_str(), check.length(), str.c_str(), str.length());

6
tests/test-tokenizer-1-spm.cpp
View File

@@ -66,9 +66,9 @@ int main(int argc, char ** argv) {
const int n_vocab = llama_n_vocab(model);
for (int i = 0; i < n_vocab; ++i) {
std::string str = llama_detokenize(ctx, std::vector<int>(1, i), true);
std::string str = common_token_to_piece(ctx, std::vector<int>(1, i), true);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false, true);
std::string check = llama_detokenize(ctx, tokens);
std::string check = common_token_to_piece(ctx, tokens);
if (check != str) {
fprintf(stderr, "%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n",
__func__, i, str.c_str(), str.length(), check.c_str(), check.length());
@@ -94,7 +94,7 @@ int main(int argc, char ** argv) {
std::string str = unicode_cpt_to_utf8(cp);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false, true);
std::string check = llama_detokenize(ctx, tokens);
std::string check = common_token_to_piece(ctx, tokens);
if (cp != 9601 && str != check) {
fprintf(stderr, "error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
cp, check.c_str(), check.length(), str.c_str(), str.length());