mirror of
https://github.com/ikawrakow/ik_llama.cpp.git
synced 2026-02-20 13:14:09 +00:00
1cb7e1bf39
* spec : add self speculative decoding and ngram-mod and refactor common : use common_ prefix for common library function llama : use LLAMA_TOKEN_NULL spec : add self speculative decoding (no draft model required) + refactor spec : add ngram-mod spec : various improvements ton ngram-map + docs spec : fix the check-rate logic of ngram-simple common : add common_speculative_is_compat() spec : simplify time measurement using common_time_meas refactor common_sampler_init refactor common_token_to_piece refactor and fix cur_p bug clean up * spec : remove check rate * spec: show warnings instead of abort --------- Co-authored-by: firecoperana <firecoperana> Co-authored-by: Sascha Rogmann <59577610+srogmann@users.noreply.github.com>
44 lines
1.2 KiB
C++
44 lines
1.2 KiB
C++
#include "ggml.h"
|
|
#include "llama.h"
|
|
#include "common.h"
|
|
#include "ngram-cache.h"
|
|
|
|
#include <cstdint>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <string>
|
|
#include <unordered_map>
|
|
#include <vector>
|
|
|
|
int main(int argc, char ** argv){
|
|
gpt_params params;
|
|
|
|
if (!gpt_params_parse(argc, argv, params)) {
|
|
gpt_params_print_usage(argc, argv, params);
|
|
return 1;
|
|
}
|
|
|
|
// init llama.cpp
|
|
llama_backend_init();
|
|
llama_numa_init(params.numa);
|
|
|
|
// load the model
|
|
llama_init_result llama_init = llama_init_from_gpt_params(params);
|
|
|
|
llama_model * model = llama_init.model;
|
|
llama_context * ctx = llama_init.context;
|
|
GGML_ASSERT(model != nullptr);
|
|
|
|
// tokenize the prompt
|
|
std::vector<llama_token> inp;
|
|
inp = ::common_tokenize(ctx, params.prompt, true, true);
|
|
fprintf(stderr, "%s: tokenization done\n", __func__);
|
|
|
|
|
|
common_ngram_cache ngram_cache;
|
|
common_ngram_cache_update(ngram_cache, LLAMA_NGRAM_STATIC, LLAMA_NGRAM_STATIC, inp, inp.size(), true);
|
|
fprintf(stderr, "%s: hashing done, writing file to %s\n", __func__, params.speculative.lookup_cache_static.c_str());
|
|
|
|
common_ngram_cache_save(ngram_cache, params.speculative.lookup_cache_static);
|
|
}
|