ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-04-29 10:51:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Correctly accumulate adaptive_p sampling time

Browse Source
This commit is contained in:
Kawrakow
2026-01-19 10:00:19 +00:00
parent 4df3251b12
commit bd2434945d
5 changed files with 25 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/sampling.cpp
View File

@@ -473,7 +473,7 @@ static llama_token llama_sampling_sample_impl(
id = llama_sample_token_mirostat_v2(ctx_main, &cur_p, mirostat_tau, mirostat_eta, &ctx_sampling->mirostat_mu); id = llama_sample_token_mirostat_v2(ctx_main, &cur_p, mirostat_tau, mirostat_eta, &ctx_sampling->mirostat_mu);
} else if (adaptive_target >= 0.0f && ctx_sampling->adapt_p_ctx!=nullptr) { } else if (adaptive_target >= 0.0f && ctx_sampling->adapt_p_ctx!=nullptr) {
// adaptive p sampling // adaptive p sampling
llama_prep_adaptive_p(&cur_p, ctx_sampling->adapt_p_ctx); llama_prep_adaptive_p(ctx_main, &cur_p, ctx_sampling->adapt_p_ctx);
sampler_queue(ctx_main, params, ctx_sampling, cur_p, std::max(1, params.min_keep)); sampler_queue(ctx_main, params, ctx_sampling, cur_p, std::max(1, params.min_keep));
id = llama_sample_token_adaptive_p(ctx_main, &cur_p, ctx_sampling->adapt_p_ctx); id = llama_sample_token_adaptive_p(ctx_main, &cur_p, ctx_sampling->adapt_p_ctx);
} else { } else {

9
include/llama.h
View File

@@ -1389,15 +1389,14 @@ LLAMA_API struct llama_grammar* llama_sampler_init_grammar_lazy_patterns(
const float decay, const float decay,
const uint32_t seed); const uint32_t seed);
void llama_prep_adaptive_p( void llama_prep_adaptive_p(struct llama_context * ctx,
llama_token_data_array * candidates, llama_token_data_array * candidates,
struct llama_sampler_adaptive_p * adapt_p_ctx); struct llama_sampler_adaptive_p * adapt_p_ctx);
/// @details Adaptive p sampler described in https://github.com/MrJackSpade/adaptive-p-docs/blob/main/README.md /// @details Adaptive p sampler described in https://github.com/MrJackSpade/adaptive-p-docs/blob/main/README.md
void llama_sample_adaptive_p( void llama_sample_adaptive_p(struct llama_context * ctx,
struct llama_context * ctx, llama_token_data_array * candidates,
llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx);
struct llama_sampler_adaptive_p * adapt_p_ctx);
/// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.

16
src/llama-sampling.cpp
View File

@@ -1061,25 +1061,28 @@ llama_token llama_sample_token_adaptive_p_impl(
const size_t idx = std::distance(ctx->cum_probs.begin(), iter); const size_t idx = std::distance(ctx->cum_probs.begin(), iter);
llama_token id = candidates->data[idx].id; llama_token id = candidates->data[idx].id;
smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
smpl->n_sample++;
GGML_ASSERT(id < int(ctx->orig_prob.size())); GGML_ASSERT(id < int(ctx->orig_prob.size()));
if (auto update_prob = ctx->orig_prob[id]; update_prob > 0) { if (auto update_prob = ctx->orig_prob[id]; update_prob > 0) {
ctx->weighted_sum = ctx->decay * ctx->weighted_sum + update_prob; ctx->weighted_sum = ctx->decay * ctx->weighted_sum + update_prob;
ctx->total_weight = ctx->decay * ctx->total_weight + 1.0f; ctx->total_weight = ctx->decay * ctx->total_weight + 1.0f;
} }
smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
smpl->n_sample++;
return id; return id;
} }
void llama_sample_adaptive_p_impl(llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx) { void llama_sample_adaptive_p_impl(struct llama_sampling * ctx, llama_token_data_array * candidates,
struct llama_sampler_adaptive_p * adapt_p_ctx) {
if (adapt_p_ctx->target < 0.0f) { if (adapt_p_ctx->target < 0.0f) {
// sampler is disabled // sampler is disabled
llama_sample_softmax_impl(nullptr, candidates); llama_sample_softmax_impl(nullptr, candidates);
return; return;
} }
auto t_start = ggml_time_us();
// incomplete softmax because final division can be fused // incomplete softmax because final division can be fused
float max_l = candidates->data[0].logit; float max_l = candidates->data[0].logit;
if (!candidates->sorted) { if (!candidates->sorted) {
@@ -1120,12 +1123,16 @@ void llama_sample_adaptive_p_impl(llama_token_data_array * candidates, struct ll
} }
candidates->sorted = false; candidates->sorted = false;
adapt_p_ctx->max_xform_logit = max_logit; adapt_p_ctx->max_xform_logit = max_logit;
ctx->t_sample_us += ggml_time_us() - t_start;
} }
void llama_prep_adaptive_p_impl( void llama_prep_adaptive_p_impl(
struct llama_sampling * smpl,
llama_token_data_array * candidates, llama_token_data_array * candidates,
struct llama_sampler_adaptive_p * adapt_p_ctx) { struct llama_sampler_adaptive_p * adapt_p_ctx) {
constexpr float kDelta = 16.6f; constexpr float kDelta = 16.6f;
auto t_start = ggml_time_us();
auto & orig_prob = adapt_p_ctx->orig_prob; auto & orig_prob = adapt_p_ctx->orig_prob;
if (candidates->size != orig_prob.size() || candidates->sorted) { if (candidates->size != orig_prob.size() || candidates->sorted) {
LLAMA_LOG_ERROR("%s: this function must be called before any other sampler has been applied\n", __func__); LLAMA_LOG_ERROR("%s: this function must be called before any other sampler has been applied\n", __func__);
@@ -1146,6 +1153,7 @@ void llama_prep_adaptive_p_impl(
orig_prob[j] = prob; orig_prob[j] = prob;
} }
adapt_p_ctx->cum_orig_prob = cum_prob; adapt_p_ctx->cum_orig_prob = cum_prob;
if (smpl) smpl->t_sample_us += ggml_time_us() - t_start;
} }
struct llama_sampler_adaptive_p * llama_init_adaptive_p_impl(int n_vocab, struct llama_sampler_adaptive_p * llama_init_adaptive_p_impl(int n_vocab,

2
src/llama-sampling.h
View File

@@ -89,10 +89,12 @@ struct llama_sampler_adaptive_p * llama_init_adaptive_p_impl(int n_vocab,
const uint32_t seed); const uint32_t seed);
void llama_prep_adaptive_p_impl( void llama_prep_adaptive_p_impl(
struct llama_sampling * smpl,
llama_token_data_array * candidates, llama_token_data_array * candidates,
struct llama_sampler_adaptive_p * adapt_p_ctx); struct llama_sampler_adaptive_p * adapt_p_ctx);
void llama_sample_adaptive_p_impl( void llama_sample_adaptive_p_impl(
struct llama_sampling * smpl,
llama_token_data_array * candidates, llama_token_data_array * candidates,
struct llama_sampler_adaptive_p * adapt_p_ctx); struct llama_sampler_adaptive_p * adapt_p_ctx);

13
src/llama.cpp
View File

@@ -7687,15 +7687,14 @@ void llama_sample_dry([[maybe_unused]] struct llama_context* ctx, struct llama_s
llama_sampler_dry_apply(smpl, candidates_p); llama_sampler_dry_apply(smpl, candidates_p);
} }
void llama_sample_adaptive_p( void llama_sample_adaptive_p(llama_context * ctx,
[[maybe_unused]] struct llama_context * ctx, llama_token_data_array * candidates,
llama_token_data_array * candidates, llama_sampler_adaptive_p * adapt_p_ctx) {
struct llama_sampler_adaptive_p * adapt_p_ctx) { llama_sample_adaptive_p_impl(&ctx->sampling, candidates, adapt_p_ctx);
llama_sample_adaptive_p_impl(candidates, adapt_p_ctx);
} }
void llama_prep_adaptive_p(llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx) { void llama_prep_adaptive_p(struct llama_context * ctx, llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx) {
llama_prep_adaptive_p_impl(candidates, adapt_p_ctx); llama_prep_adaptive_p_impl(&ctx->sampling, candidates, adapt_p_ctx);
} }