ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-02-28 17:14:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

softcap: minor improvement (#24)

Browse Source 
Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
This commit is contained in:
Kawrakow
2024-08-21 13:00:09 +03:00
committed by GitHub
parent d259a50ca6
commit bd99ed7d0a
2 changed files with 55 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/llama.cpp
View File

@@ -8325,9 +8325,6 @@ static struct ggml_tensor * llm_build_kqv(
if (hparams.attn_soft_cap) {
kq = ggml_softcap(ctx, kq, 1.0f / hparams.f_attn_logit_softcapping, hparams.f_attn_logit_softcapping);
//kq = ggml_scale(ctx, kq, 1.0f / hparams.f_attn_logit_softcapping);
//kq = ggml_tanh(ctx, kq);
//kq = ggml_scale(ctx, kq, hparams.f_attn_logit_softcapping);
}
kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias);