ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-04-27 01:49:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

spec : add self speculative decoding, ngram and refactor (#1261)

Browse Source 
* spec : add self speculative decoding and ngram-mod and refactor

common : use common_ prefix for common library function

llama : use LLAMA_TOKEN_NULL

spec : add self speculative decoding (no draft model required) + refactor

spec : add ngram-mod

spec : various improvements ton ngram-map + docs

spec : fix the check-rate logic of ngram-simple

common : add common_speculative_is_compat()

spec : simplify time measurement using common_time_meas

refactor common_sampler_init

refactor common_token_to_piece

refactor and fix cur_p bug

clean up

* spec : remove check rate

* spec: show warnings instead of abort

---------

Co-authored-by: firecoperana <firecoperana>
Co-authored-by: Sascha Rogmann <59577610+srogmann@users.noreply.github.com>
This commit is contained in:
firecoperana
2026-02-13 12:04:55 -06:00
committed by GitHub
parent 1fdbc0dafe
commit 1cb7e1bf39
54 changed files with 2652 additions and 779 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
examples/server/server-context.h
View File

@@ -29,6 +29,9 @@ struct server_slot {
struct slot_params params;
llama_batch batch_spec = {};
llama_context * ctx_dft = nullptr;
slot_state state = SLOT_STATE_IDLE;
slot_command command = SLOT_COMMAND_NONE;
@@ -103,8 +106,6 @@ struct server_slot {
llama_token sampled; // in speculative mode, this is the last accepted token
llama_tokens drafted;
struct llama_sampling_params sparams;
llama_sampling_context* ctx_sampling = nullptr;
json json_schema;
common_chat_format chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
@@ -121,9 +122,9 @@ struct server_slot {
mtmd_context* mctx = nullptr;
// speculative decoding
struct llama_speculative* spec = nullptr;
llama_context* ctx_dft = nullptr;
llama_batch batch_spec = {};
struct common_speculative * spec = nullptr;
struct common_params_sampling sparams;
common_sampler * ctx_sampling = nullptr;
// speculative decoding stats
int32_t n_draft_total = 0; // Total draft tokens generated
@@ -151,6 +152,8 @@ struct server_slot {
void add_token_string(const completion_token_output& token);
bool can_speculate() const;
int get_n_draft_max() const;
void release();
@@ -164,6 +167,7 @@ struct server_slot {
size_t find_stopping_strings(const std::string& text, const size_t last_token_size, bool is_full_stop);
void print_timings() const;
};
struct server_metrics {