ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-04-30 19:31:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

bitnet: add 2 bpw quantization

Browse Source 
The scalar dot product already chieves 37 t/s for TG!
This commit is contained in:
Iwan Kawrakow
2024-06-17 18:41:30 +03:00
parent 765622ff8f
commit f6863cfa1b
10 changed files with 216 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
ggml-common.h
View File

@@ -380,6 +380,14 @@ typedef struct {
uint8_t qh[QK_IQ1BN/16];
} block_iq1_bn;
static_assert(sizeof(block_iq1_bn) == sizeof(uint16_t) + QK_IQ1BN/8 + QK_IQ1BN/16, "wrong iq1_bn block size/padding");
//
// Bitnet - implemented as 2.0 bpw
//
#define QK_IQ2BN 64
typedef struct {
uint8_t qs[QK_IQ2BN/4];
} block_iq2_bn;
static_assert(sizeof(block_iq2_bn) == QK_IQ2BN/4, "wrong iq2_bn block size/padding");
// Used by IQ1_M quants
typedef union {