ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-02-20 13:14:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Custom quantization rules with regular expressions (#244)

Browse Source 
* Custom quantization rules with regular expressions

* Add the --custom-q option to the help

---------

Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
This commit is contained in:
Kawrakow
2025-03-07 08:54:09 +02:00
committed by GitHub
parent 7bdbf99bbd
commit c67a37b251
3 changed files with 51 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/llama.cpp
View File

@@ -16283,6 +16283,19 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n
return i_layer < n_layers/8 || i_layer >= 7*n_layers/8 || (i_layer - n_layers/8)%3 == 2;
};
auto custom_type = GGML_TYPE_COUNT;
if (qs.params->custom_quants) {
using CustomQ = std::pair<std::string, ggml_type>;
auto& q_rules = *static_cast<const std::vector<CustomQ>*>(qs.params->custom_quants);
for (auto& rule : q_rules) {
std::regex pattern(rule.first);
if (std::regex_search(name, pattern)) {
custom_type = rule.second;
break;
}
}
}
//auto get_layer = [] (const char * name) {
// int il;
// if (sscanf(name, "blk.%d.", &il) == 1) return il;
@@ -16752,6 +16765,11 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n
++qs.i_ffn_up;
}
if (custom_type < GGML_TYPE_COUNT) {
new_type = custom_type;
LLAMA_LOG_INFO("Using custom type %s for tensor %s\n", ggml_type_name(new_type), name.c_str());
}
// if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;
//}
// IK: let's remove this, else Q2_K is almost the same as Q3_K_S
@@ -17791,6 +17809,7 @@ struct llama_model_quantize_params llama_model_quantize_default_params() {
/*.ignore_imatrix_rules =*/ false,
/*.imatrix =*/ nullptr,
/*.kv_overrides =*/ nullptr,
/*.custom_quants =*/ nullptr,
};
return result;