ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-02-25 07:34:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use fused mul - unary op also for MoE models

Browse Source
This commit is contained in:
Iwan Kawrakow
2024-10-26 19:22:02 +03:00
parent 2e5f6db5de
commit fe767a45ac
1 changed files with 17 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/llama.cpp
View File

@@ -8326,22 +8326,24 @@ static struct ggml_tensor * llm_build_moe_ffn(
ggml_tensor * gate = llm_build_lora_mm_id(lctx, ctx, gate_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens]
cb(gate, "ffn_moe_gate", il);
switch (type_op) {
case LLM_FFN_SILU:
{
gate = ggml_silu(ctx, gate);
cb(gate, "ffn_moe_silu", il);
} break;
case LLM_FFN_GELU:
{
gate = ggml_gelu(ctx, gate);
cb(gate, "ffn_moe_gelu", il);
} break;
default:
GGML_ABORT("fatal error");
}
// This is equivalent to the commented out code below
ggml_tensor * par = ggml_fused_mul_unary(ctx, gate, up, type_op == LLM_FFN_SILU ? GGML_UNARY_OP_SILU : GGML_UNARY_OP_GELU);
ggml_tensor * par = ggml_mul(ctx, up, gate); // [n_ff, n_expert_used, n_tokens]
//switch (type_op) {
// case LLM_FFN_SILU:
// {
// gate = ggml_silu(ctx, gate);
// cb(gate, "ffn_moe_silu", il);
// } break;
// case LLM_FFN_GELU:
// {
// gate = ggml_gelu(ctx, gate);
// cb(gate, "ffn_moe_gelu", il);
// } break;
// default:
// GGML_ABORT("fatal error");
//}
//ggml_tensor * par = ggml_mul(ctx, up, gate); // [n_ff, n_expert_used, n_tokens]
cb(par, "ffn_moe_gate_par", il);
ggml_tensor * experts = llm_build_lora_mm_id(lctx, ctx, down_exps, par, selected_experts); // [n_embd, n_expert_used, n_tokens]