From a96e5449cc7b17f75f57af5f4e3d2510ac897cc7 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Mon, 19 Jan 2026 10:17:07 +0000 Subject: [PATCH] Correctly accumulate sampling time for adaptive_p --- common/sampling.cpp | 2 +- include/llama.h | 2 +- src/llama-sampling.cpp | 18 +++++++++++++----- src/llama-sampling.h | 2 ++ src/llama.cpp | 6 +++--- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index f778b159..f68fdc0e 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -471,7 +471,7 @@ static llama_token llama_sampling_sample_impl( id = llama_sample_token_mirostat_v2(ctx_main, &cur_p, mirostat_tau, mirostat_eta, &ctx_sampling->mirostat_mu); } else if (adaptive_target >= 0.0f && ctx_sampling->adapt_p_ctx!=nullptr) { // adaptive p sampling - llama_prep_adaptive_p(&cur_p, ctx_sampling->adapt_p_ctx); + llama_prep_adaptive_p(ctx_main, &cur_p, ctx_sampling->adapt_p_ctx); sampler_queue(ctx_main, params, ctx_sampling, cur_p, std::max(1, params.min_keep)); id = llama_sample_token_adaptive_p(ctx_main, &cur_p, ctx_sampling->adapt_p_ctx); } else { diff --git a/include/llama.h b/include/llama.h index 3d17f9b2..dd0bb409 100644 --- a/include/llama.h +++ b/include/llama.h @@ -1389,7 +1389,7 @@ LLAMA_API struct llama_grammar* llama_sampler_init_grammar_lazy_patterns( const float decay, const uint32_t seed); - void llama_prep_adaptive_p( + void llama_prep_adaptive_p(struct llama_context * ctx, llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx); diff --git a/src/llama-sampling.cpp b/src/llama-sampling.cpp index 17f95f0a..acfcefd4 100644 --- a/src/llama-sampling.cpp +++ b/src/llama-sampling.cpp @@ -1061,15 +1061,15 @@ llama_token llama_sample_token_adaptive_p_impl( const size_t idx = std::distance(ctx->cum_probs.begin(), iter); llama_token id = candidates->data[idx].id; - smpl->t_sample_us += ggml_time_us() - t_start_sample_us; - smpl->n_sample++; - if (auto it = ctx->orig_prob_map.find(id); it != ctx->orig_prob_map.end()) { float update_prob = it->second / ctx->cum_orig_prob; ctx->weighted_sum = ctx->decay * ctx->weighted_sum + update_prob; ctx->total_weight = ctx->decay * ctx->total_weight + 1.0f; } + smpl->t_sample_us += ggml_time_us() - t_start_sample_us; + smpl->n_sample++; + //float update_prob = candidates->data[idx].p; // not ideal //if (ctx->orig_prob_map.contains(id)) { // // selected token id is among tracked ids @@ -1083,13 +1083,16 @@ llama_token llama_sample_token_adaptive_p_impl( return id; } -void llama_sample_adaptive_p_impl(llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx) { +void llama_sample_adaptive_p_impl(struct llama_sampling * ctx, llama_token_data_array * candidates, + struct llama_sampler_adaptive_p * adapt_p_ctx) { if (adapt_p_ctx->target < 0.0f) { // sampler is disabled llama_sample_softmax_impl(nullptr, candidates); return; } + auto t_start = ggml_time_us(); + // incomplete softmax because final division can be fused float max_l = candidates->data[0].logit; if (!candidates->sorted) { @@ -1130,12 +1133,15 @@ void llama_sample_adaptive_p_impl(llama_token_data_array * candidates, struct ll } candidates->sorted = false; adapt_p_ctx->max_xform_logit = max_logit; + + ctx->t_sample_us += ggml_time_us() - t_start; } -void llama_prep_adaptive_p_impl( +void llama_prep_adaptive_p_impl(struct llama_sampling * smpl, llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx) { constexpr float kDelta = 16.6f; + auto t_start = ggml_time_us(); if (!candidates->sorted) { float max_logit = candidates->data[0].logit; for (int j = 1; j < int(candidates->size); ++j) { @@ -1152,6 +1158,7 @@ void llama_prep_adaptive_p_impl( } } adapt_p_ctx->cum_orig_prob = cum_prob; + if (smpl) smpl->t_sample_us += ggml_time_us() - t_start; return; } @@ -1169,6 +1176,7 @@ void llama_prep_adaptive_p_impl( adapt_p_ctx->orig_prob_map[candidates->data[j].id] = prob; } adapt_p_ctx->cum_orig_prob = cum_prob; + if (smpl) smpl->t_sample_us += ggml_time_us() - t_start; //if (!candidates->sorted) { // std::sort(candidates->data, candidates->data + candidates->size, diff --git a/src/llama-sampling.h b/src/llama-sampling.h index 55b4371c..3249c843 100644 --- a/src/llama-sampling.h +++ b/src/llama-sampling.h @@ -89,10 +89,12 @@ struct llama_sampler_adaptive_p * llama_init_adaptive_p_impl( const uint32_t seed); void llama_prep_adaptive_p_impl( + struct llama_sampling * smpl, llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx); void llama_sample_adaptive_p_impl( + struct llama_sampling * smpl, llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx); diff --git a/src/llama.cpp b/src/llama.cpp index 767ab687..d76ac5b7 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -7691,11 +7691,11 @@ void llama_sample_adaptive_p( [[maybe_unused]] struct llama_context * ctx, llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx) { - llama_sample_adaptive_p_impl(candidates, adapt_p_ctx); + llama_sample_adaptive_p_impl(&ctx->sampling, candidates, adapt_p_ctx); } -void llama_prep_adaptive_p(llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx) { - llama_prep_adaptive_p_impl(candidates, adapt_p_ctx); +void llama_prep_adaptive_p(struct llama_context * ctx, llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx) { + llama_prep_adaptive_p_impl(&ctx->sampling, candidates, adapt_p_ctx); }