diff --git a/common/common.cpp b/common/common.cpp index df4d5d0f..7c0bd16d 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -994,8 +994,8 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa params.cont_batching = false; return true; } - if (arg == "-fa" || arg == "--flash-attn") { - params.flash_attn = true; + if (arg == "-no-fa" || arg == "--no-flash-attn") { + params.flash_attn = false; return true; } if (arg == "-mla" || arg == "--mla-use") { @@ -1008,8 +1008,8 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa params.attn_max_batch = std::stoi(argv[i]); return true; } - if (arg == "-fmoe" || arg == "--fused-moe") { - params.fused_moe_up_gate = true; + if (arg == "-no-fmoe" || arg == "--no-fused-moe") { + params.fused_moe_up_gate = false; return true; } if (arg == "-ger" || arg == "--grouped-expert-routing") { @@ -1804,10 +1804,10 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param options.push_back({ "*", "-ub, --ubatch-size N", "physical maximum batch size (default: %d)", params.n_ubatch }); options.push_back({ "*", " --keep N", "number of tokens to keep from the initial prompt (default: %d, -1 = all)", params.n_keep }); options.push_back({ "*", " --chunks N", "max number of chunks to process (default: %d, -1 = all)", params.n_chunks }); - options.push_back({ "*", "-fa, --flash-attn", "enable Flash Attention (default: %s)", params.flash_attn ? "enabled" : "disabled" }); + options.push_back({ "*", "-no-fa, --no-flash-attn", "disable Flash Attention (default: %s)", params.flash_attn ? "enabled" : "disabled" }); options.push_back({ "*", "-mla, --mla-use", "enable MLA (default: %d)", params.mla_attn }); options.push_back({ "*", "-amb, --attention-max-batch", "max batch size for attention computations (default: %d)", params.attn_max_batch}); - options.push_back({ "*", "-fmoe, --fused-moe", "enable fused MoE (default: %s)", params.fused_moe_up_gate ? "enabled" : "disabled" }); + options.push_back({ "*", "-no-fmoe, --no-fused-moe", "disable fused MoE (default: %s)", params.fused_moe_up_gate ? "enabled" : "disabled" }); options.push_back({ "*", "-ger, --grouped-expert-routing", "enable grouped expert routing (default: %s)", params.grouped_expert_routing ? "enabled" : "disabled" }); options.push_back({ "*", "-no-fug, --no-fused-up-gate", "disaable fused up-gate (default: %s)", params.fused_up_gate ? "enabled" : "disabled" }); options.push_back({ "*", "-no-mmad, --no-fused-mul-multiadd", "disaable fused mul-multi_add (default: %s)", params.fused_mmad? "enabled" : "disabled" }); diff --git a/common/common.h b/common/common.h index 327afb24..f41e7580 100644 --- a/common/common.h +++ b/common/common.h @@ -230,10 +230,10 @@ struct gpt_params { bool multiline_input = false; // reverse the usage of `\` bool simple_io = false; // improves compatibility with subprocesses and limited consoles bool cont_batching = true; // insert new sequences for decoding on-the-fly - bool flash_attn = false; // flash attention + bool flash_attn = true; // flash attention int mla_attn = 0; // MLA 0: standard attention, 1: MLA with K and transposed V cache, 2: MLA with just K cache int attn_max_batch = 0; // Max batch size to use when computing attention (only applicable if flash_attn = false) - bool fused_moe_up_gate = false; // fused up*unary(gate) op for MoE models + bool fused_moe_up_gate = true; // fused up*unary(gate) op for MoE models bool fused_up_gate = true; // fused up*unary(gate) op bool fused_mmad = true; // fused mul+multi_add op bool grouped_expert_routing = false; // if to use grouped expert routing (BailingMoeV2 arch) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 9a6a27f1..3e948e68 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -260,7 +260,7 @@ struct cmd_params { bool verbose; bool warmup; bool repack = false; - bool fmoe = false; + bool fmoe = true; bool ger = false; // ger = Grouped Expert Routing bool no_fug = false; bool use_thp = false; @@ -285,7 +285,7 @@ static const cmd_params cmd_params_defaults = { /* split_mode */ {LLAMA_SPLIT_MODE_LAYER}, /* main_gpu */ {0}, /* no_kv_offload */ {false}, - /* flash_attn */ {false}, + /* flash_attn */ {true}, /* mla_attn */ {0}, /* attn_max_batch */ {0}, /* ser */ {{-1,0.0f}}, @@ -298,7 +298,7 @@ static const cmd_params cmd_params_defaults = { /* verbose */ false, /* warmup */ true, /* repack */ false, - /* fmoe */ false, + /* fmoe */ true, /* ger */ false, /* no_fug */ false, /* use_thp */ false, @@ -846,7 +846,7 @@ struct cmd_params_instance { bool use_mmap; bool embeddings; bool repack = false; - bool fmoe = false; + bool fmoe = true; bool ger = false; bool no_fug = false; bool use_thp = false; diff --git a/src/llama.cpp b/src/llama.cpp index fe3bfbce..2cce5384 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -3750,10 +3750,10 @@ struct llama_context_params llama_context_default_params() { /*.logits_all =*/ false, /*.embeddings =*/ false, /*.offload_kqv =*/ true, - /*.flash_attn =*/ false, + /*.flash_attn =*/ true, /*.mla_attn =*/ 0, /*.attn_max_batch =*/ 0, - /*.fused_moe_up_gate =*/ false, + /*.fused_moe_up_gate =*/ true, /*.grouped_expert_routing =*/ false, /*.fused_up_gate =*/ true, /*.fused_mmad =*/ true, @@ -4040,19 +4040,19 @@ struct llama_context * llama_new_context_with_model( cparams.mla_attn = 0; } - LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx); - LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch); - LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch); - LLAMA_LOG_INFO("%s: flash_attn = %d\n", __func__, cparams.flash_attn); - LLAMA_LOG_INFO("%s: mla_attn = %d\n", __func__, cparams.mla_attn); - LLAMA_LOG_INFO("%s: attn_max_b = %d\n", __func__, cparams.attn_max_batch); - LLAMA_LOG_INFO("%s: fused_moe = %d\n", __func__, cparams.fused_moe_up_gate); - LLAMA_LOG_INFO("%s: grouped er = %d\n", __func__, cparams.grouped_expert_routing); + LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx); + LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch); + LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch); + LLAMA_LOG_INFO("%s: flash_attn = %d\n", __func__, cparams.flash_attn); + LLAMA_LOG_INFO("%s: mla_attn = %d\n", __func__, cparams.mla_attn); + LLAMA_LOG_INFO("%s: attn_max_b = %d\n", __func__, cparams.attn_max_batch); + LLAMA_LOG_INFO("%s: fused_moe = %d\n", __func__, cparams.fused_moe_up_gate); + LLAMA_LOG_INFO("%s: grouped er = %d\n", __func__, cparams.grouped_expert_routing); LLAMA_LOG_INFO("%s: fused_up_gate = %d\n", __func__, cparams.fused_up_gate); - LLAMA_LOG_INFO("%s: fused_mmad = %d\n", __func__, cparams.fused_mmad); - LLAMA_LOG_INFO("%s: ser = %d, %g\n", __func__, cparams.min_experts, cparams.thresh_experts); - LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base); - LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale); + LLAMA_LOG_INFO("%s: fused_mmad = %d\n", __func__, cparams.fused_mmad); + LLAMA_LOG_INFO("%s: ser = %d, %g\n", __func__, cparams.min_experts, cparams.thresh_experts); + LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base); + LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale); ctx->abort_callback = params.abort_callback; ctx->abort_callback_data = params.abort_callback_data;