ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-04-28 10:21:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

iq3_k: Basics

Browse Source 
Quantize/dequantize, CUDA dequantize.
PPL of LLaMA-3.1-8B is better than iq3_s and iq3_m.
This commit is contained in:
Iwan Kawrakow
2024-07-30 16:11:25 +03:00
parent 92b3e1c034
commit c828bd3c57
15 changed files with 416 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
ggml/include/ggml.h
View File

@@ -390,8 +390,9 @@ extern "C" {
GGML_TYPE_IQ2_BN = 35,
GGML_TYPE_Q8_K64 = 36,
GGML_TYPE_IQ2_K = 37,
GGML_TYPE_IQ4_K = 38,
GGML_TYPE_IQ5_K = 39,
GGML_TYPE_IQ3_K = 38,
GGML_TYPE_IQ4_K = 39,
GGML_TYPE_IQ5_K = 40,
GGML_TYPE_COUNT,
};
@@ -439,8 +440,9 @@ extern "C" {
GGML_FTYPE_MOSTLY_IQ1_BN = 28, // except 1d tensors
GGML_FTYPE_MOSTLY_IQ2_BN = 29, // except 1d tensors
GGML_FTYPE_MOSTLY_IQ2_K = 30, // except 1d tensors
GGML_FTYPE_MOSTLY_IQ4_K = 31, // except 1d tensors
GGML_FTYPE_MOSTLY_IQ5_K = 32, // except 1d tensors
GGML_FTYPE_MOSTLY_IQ3_K = 31, // except 1d tensors
GGML_FTYPE_MOSTLY_IQ4_K = 32, // except 1d tensors
GGML_FTYPE_MOSTLY_IQ5_K = 33, // except 1d tensors
};
// available tensor operations: