ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-01-26 17:20:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Specify tensor name regex for tensors to be repacked (#274)

Browse Source 
Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
This commit is contained in:
Kawrakow
2025-03-21 10:51:37 +01:00
committed by GitHub
parent 24e780ba74
commit 4158743014
3 changed files with 52 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
examples/quantize/quantize.cpp
View File

@@ -146,6 +146,7 @@ static void usage(const char * executable) {
printf(" --token-embedding-type ggml_type: use this ggml_type for the token_embd.weight tensor.\n\n");
printf(" --custom-q regex1=type1,regex2=type2...: use this to specify custom quantization type rules.\n\n");
printf(" --repack Repack all tensors to the corresponding _r4/8 variant if available.\n\n");
printf(" --repack-pattern Comma separated list of regexs to use for matching tensor names to be repacked.\n\n");
printf("Additional specific tensor quantization types used in the custom quant scheme 'CQS (default is Q2_K):\n");
printf(" --attn-q-type ggml_type: use this ggml_type for the attn_q.weight tensor.\n");
printf(" --attn-k-type ggml_type: use this ggml_type for the attn_k.weight tensor.\n");
@@ -327,6 +328,8 @@ int main(int argc, char ** argv) {
std::vector<llama_model_kv_override> kv_overrides;
std::vector<CustomQ> custom_quants;
std::vector<std::string> repack_patterns;
for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) {
if (strcmp(argv[arg_idx], "--leave-output-tensor") == 0) {
params.quantize_output_tensor = false;
@@ -334,6 +337,13 @@ int main(int argc, char ** argv) {
params.ignore_imatrix_rules = true;
} else if (strcmp(argv[arg_idx], "--repack") == 0) {
params.only_repack = true;
} else if (strcmp(argv[arg_idx], "--repack-pattern") == 0) {
if (arg_idx < argc-1) {
auto p = string_split(argv[++arg_idx], ',');
repack_patterns.insert(repack_patterns.end(), p.begin(), p.end());
} else {
usage(argv[0]);
}
} else if (strcmp(argv[arg_idx], "--output-tensor-type") == 0) {
if (arg_idx < argc-1) {
params.output_tensor_type = parse_ggml_type(argv[++arg_idx]);
@@ -431,6 +441,10 @@ int main(int argc, char ** argv) {
}
}
if (!repack_patterns.empty()) {
params.repack_pattern = &repack_patterns;
}
if (argc - arg_idx < 2) {
printf("%s: bad arguments\n", argv[0]);
usage(argv[0]);