ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-04-29 19:01:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

iq2_kt: SOTA

Browse Source 
We arrive at
PPL(LLaMA-3.1-8B-Instruct, 8192) = 9.0297
PPL(LLaMA-2-7B,            4096) = 6.3913

Ah, quantization is faster too. About 20% faster.
This commit is contained in:
Iwan Kawrakow
2024-11-13 11:24:16 +02:00
parent 200a19f18f
commit dbe085474a
1 changed files with 97 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

308
ggml/src/iqk/iqk_quantize.cpp
View File

@@ -22,6 +22,7 @@
#include <cstring> #include <cstring>
#include <mutex> #include <mutex>
#include <random> #include <random>
#include <memory>
#if defined(_MSC_VER) #if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data #pragma warning(disable: 4244 4267) // possible loss of data
@@ -3150,7 +3151,7 @@ __m256 hsum_float_4x8(__m256 * accm) {
return _mm256_add_ps(_mm256_unpacklo_ps(accm[0], accm[1]), _mm256_unpackhi_ps(accm[0], accm[1])); return _mm256_add_ps(_mm256_unpacklo_ps(accm[0], accm[1]), _mm256_unpackhi_ps(accm[0], accm[1]));
} }
#endif #endif
template <int block_size, int group_size, int num_bits, int num_clusters> template <int block_size, int group_size, int num_bits>
class QuantizerIQKT { class QuantizerIQKT {
static_assert(group_size == 8 || group_size == 4); static_assert(group_size == 8 || group_size == 4);
static_assert(block_size >= 8 && block_size%8 == 0); static_assert(block_size >= 8 && block_size%8 == 0);
@@ -3164,11 +3165,10 @@ public:
constexpr static float kScale = 31.75f; constexpr static float kScale = 31.75f;
constexpr static bool kVerbose = false; constexpr static bool kVerbose = false;
QuantizerIQKT(); QuantizerIQKT(int num_clusters, int num_neighbours);
const float * values() const { return m_values.data(); } const float * values() const { return m_values.data(); }
inline void find_best_match(float d, const float * xb, const float * weight, int * best_idx) const; inline void find_best_match(float d, const float * xb, const float * weight, int * best_idx) const;
inline void find_best_match(const float * xb, const float * weight, int * best_idx) const;
inline std::pair<float, float> find_best_scale(const float * xb, const float * weight, const int * best_idx) const; inline std::pair<float, float> find_best_scale(const float * xb, const float * weight, const int * best_idx) const;
inline float find_best_inverse_scale(const float * xb, const float * weight, const int * best_idx) const; inline float find_best_inverse_scale(const float * xb, const float * weight, const int * best_idx) const;
@@ -3205,17 +3205,18 @@ public:
} }
} }
private: private:
static std::vector<float> cluster_points(const std::vector<float>& points, int ncluster, int niter); static std::vector<float> cluster_points(const std::vector<float>& points, int ncluster, int niter, float * mid);
static std::vector<std::vector<int>> finalize_clusters(const std::vector<float>& points, const std::vector<float>& clusters, static std::vector<std::vector<int>> finalize_clusters(int num_neighbours, const std::vector<float>& points, const std::vector<float>& clusters,
std::vector<std::vector<float>>& c_values); std::vector<std::vector<float>>& c_values);
std::vector<float> m_values; std::vector<float> m_values;
std::vector<float> m_clusters; std::vector<float> m_clusters;
std::vector<std::vector<int>> m_in_cluster; std::vector<std::vector<int>> m_in_cluster;
std::vector<std::vector<float>> m_c_values; std::vector<std::vector<float>> m_c_values;
float m_mid[kGroupSize];
}; };
template <int block_size, int group_size, int num_bits, int num_clusters> template <int block_size, int group_size, int num_bits>
QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::QuantizerIQKT() { QuantizerIQKT<block_size, group_size, num_bits>::QuantizerIQKT(int num_clusters, int num_neighbours) {
m_values.resize(kNumVal*kGroupSize); m_values.resize(kNumVal*kGroupSize);
float * data = m_values.data(); float * data = m_values.data();
for (int i = 0; i < kNumVal; ++i) { for (int i = 0; i < kNumVal; ++i) {
@@ -3225,13 +3226,13 @@ QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::QuantizerIQKT() {
// Make 128 clusters. // Make 128 clusters.
// Note: we get a slightly better result by using 64 clusters // Note: we get a slightly better result by using 64 clusters
// at the expense of almost doubling the quantization time. // at the expense of almost doubling the quantization time.
m_clusters = cluster_points(m_values, num_clusters, 200); m_clusters = cluster_points(m_values, num_clusters, 200, m_mid);
GGML_ASSERT(!m_clusters.empty()); GGML_ASSERT(!m_clusters.empty());
m_in_cluster = finalize_clusters(m_values, m_clusters, m_c_values); m_in_cluster = finalize_clusters(num_neighbours, m_values, m_clusters, m_c_values);
} }
template <int block_size, int group_size, int num_bits, int num_clusters> template <int block_size, int group_size, int num_bits>
std::pair<float, float> QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::find_best_scale( std::pair<float, float> QuantizerIQKT<block_size, group_size, num_bits>::find_best_scale(
const float * xb, const float * weight, const int * best_idx) const { const float * xb, const float * weight, const int * best_idx) const {
float sumqx = 0, sumq2 = 0; float sumqx = 0, sumq2 = 0;
#ifdef __AVX2__ #ifdef __AVX2__
@@ -3263,8 +3264,8 @@ std::pair<float, float> QuantizerIQKT<block_size, group_size, num_bits, num_clus
return sumq2 > 0 ? std::make_pair(sumqx/sumq2, sumqx*sumqx/sumq2) : std::make_pair(0.f, 0.f); return sumq2 > 0 ? std::make_pair(sumqx/sumq2, sumqx*sumqx/sumq2) : std::make_pair(0.f, 0.f);
} }
template <int block_size, int group_size, int num_bits, int num_clusters> template <int block_size, int group_size, int num_bits>
float QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::find_best_inverse_scale( float QuantizerIQKT<block_size, group_size, num_bits>::find_best_inverse_scale(
const float * xb, const float * weight, const int * best_idx) const { const float * xb, const float * weight, const int * best_idx) const {
float sumqx = 0, sumx2 = 0; float sumqx = 0, sumx2 = 0;
#ifdef __AVX2__ #ifdef __AVX2__
@@ -3296,151 +3297,8 @@ float QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::find_best_i
return sumx2 > 0 ? sumqx/sumx2 : 0.f; return sumx2 > 0 ? sumqx/sumx2 : 0.f;
} }
template <int block_size, int group_size, int num_bits, int num_clusters> template <int block_size, int group_size, int num_bits>
void QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::find_best_match(const float * xb, const float * weight, int * best_idx) const { void QuantizerIQKT<block_size, group_size, num_bits>::find_best_match(float d, const float * xb, const float * weight, int * best_idx) const {
int ncluster = m_clusters.size()/kGroupSize;
#ifdef __AVX2__
if constexpr (kGroupSize == 8) {
__m256 sqx[8];
const __m256i add_idx = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
float sx[8];
int index[8];
for (int l = 0; l < kNg; ++l) {
auto xl = xb + 8*l;
auto wl = weight + 8*l;
auto vx = _mm256_loadu_ps(xl);
auto vw = _mm256_loadu_ps(wl);
auto vbest = _mm256_set1_ps(0.f);
auto best_index = _mm256_set1_epi32(-1);
float best = 0; int jbest = -1;
for (int j = 0; j < ncluster; j += 8) {
auto idx = _mm256_add_epi32(_mm256_set1_epi32(j), add_idx);
for (int i = 0; i < 8; ++i) {
auto vq = _mm256_loadu_ps(m_clusters.data() + kGroupSize*(j+i));
auto sumqx = _mm256_mul_ps(vw, _mm256_mul_ps(vx, vq));
auto sumq2 = hsum_float_8(_mm256_mul_ps(vw, _mm256_mul_ps(vq, vq)));
sqx[i] = _mm256_mul_ps(_mm256_set1_ps(sumq2 > 0 ? 1/sumq2 : 0), _mm256_mul_ps(sumqx, sumqx));
}
auto score = hsum_float_8x8(sqx);
auto mask = _mm256_cmp_ps(score, vbest, _CMP_GT_OQ);
best_index = _mm256_or_si256(_mm256_and_si256(_mm256_castps_si256(mask), idx),
_mm256_andnot_si256(_mm256_castps_si256(mask), best_index));
vbest = _mm256_max_ps(vbest, score);
}
_mm256_store_ps(sx, vbest);
_mm256_store_si256((__m256i *)index, best_index);
for (int i = 0; i < 8; ++i) {
if (sx[i] > best) { best = sx[i]; jbest = index[i]; }
}
auto& points = m_in_cluster[jbest];
auto& values = m_c_values[jbest];
GGML_ASSERT(!points.empty() && points.size()%8 == 0);
int jbest_cluster = jbest;
vbest = _mm256_set1_ps(0.f);
best_index = _mm256_set1_epi32(-1);
best = 0; jbest = -1;
for (int j = 0; j < int(points.size()); j += 8) {
auto idx = _mm256_add_epi32(_mm256_set1_epi32(j), add_idx);
for (int i = 0; i < 8; ++i) {
auto vq = _mm256_loadu_ps(values.data() + kGroupSize*(j+i));
auto sumqx = _mm256_mul_ps(vw, _mm256_mul_ps(vx, vq));
auto sumq2 = hsum_float_8(_mm256_mul_ps(vw, _mm256_mul_ps(vq, vq)));
sqx[i] = _mm256_mul_ps(_mm256_set1_ps(sumq2 > 0 ? 1/sumq2 : 0), _mm256_mul_ps(sumqx, sumqx));
}
auto score = hsum_float_8x8(sqx);
auto mask = _mm256_cmp_ps(score, vbest, _CMP_GT_OQ);
best_index = _mm256_or_si256(_mm256_and_si256(_mm256_castps_si256(mask), idx),
_mm256_andnot_si256(_mm256_castps_si256(mask), best_index));
vbest = _mm256_max_ps(vbest, score);
}
_mm256_store_ps(sx, vbest);
_mm256_store_si256((__m256i *)index, best_index);
for (int i = 0; i < 8; ++i) {
if (sx[i] > best) { best = sx[i]; jbest = index[i]; }
}
if (jbest < 0) {
fprintf(stderr, "Oops: jbest = %d for cluster %d with %d points\n", jbest, jbest_cluster, int(points.size()));
GGML_ASSERT(false);
}
best_idx[l] = points[jbest];
}
} else {
__m128 sqx[4];
const __m128i add_idx = _mm_set_epi32(3, 2, 1, 0);
float sx[4];
int index[4];
for (int l = 0; l < kNg; ++l) {
auto xl = xb + 4*l;
auto wl = weight + 4*l;
auto vx = _mm_loadu_ps(xl);
auto sumx2 = hsum_float_4(_mm_mul_ps(vx, vx));
if (!sumx2) {
best_idx[l] = 0; continue;
}
auto vw = _mm_loadu_ps(wl);
auto vbest = _mm_set1_ps(0);
auto best_index = _mm_set1_epi32(-1);
float best = 0; int jbest = -1;
for (int j = 0; j < ncluster; j += 4) {
auto idx = _mm_add_epi32(_mm_set1_epi32(j), add_idx);
for (int i = 0; i < 4; ++i) {
auto vq = _mm_loadu_ps(m_clusters.data() + kGroupSize*(j+i));
auto sumqx = _mm_mul_ps(vw, _mm_mul_ps(vx, vq));
auto sumq2 = hsum_float_4(_mm_mul_ps(vw, _mm_mul_ps(vq, vq)));
sqx[i] = _mm_mul_ps(_mm_set1_ps(sumq2 > 0 ? 1/sumq2 : 0), _mm_mul_ps(sumqx, sumqx));
}
auto score = hsum_float_4x4(sqx);
auto mask = _mm_cmp_ps(score, vbest, _CMP_GT_OQ);
best_index = _mm_or_si128(_mm_and_si128(_mm_castps_si128(mask), idx),
_mm_andnot_si128(_mm_castps_si128(mask), best_index));
vbest = _mm_max_ps(vbest, score);
}
_mm_store_ps(sx, vbest);
_mm_store_si128((__m128i *)index, best_index);
for (int i = 0; i < 4; ++i) {
if (sx[i] > best) { best = sx[i]; jbest = index[i]; }
}
GGML_ASSERT(jbest >= 0 && jbest <= int(m_in_cluster.size()));
auto& points = m_in_cluster[jbest];
GGML_ASSERT(!points.empty() && points.size()%4 == 0);
int jbest_cluster = jbest;
vbest = _mm_set1_ps(0);
best_index = _mm_set1_epi32(-1);
best = 0; jbest = -1;
for (int j = 0; j < int(points.size()); j += 4) {
auto idx = _mm_loadu_si128((const __m128i*)(points.data() + j));
for (int i = 0; i < 4; ++i) {
auto vq = _mm_loadu_ps(m_values.data() + kGroupSize*points[j+i]);
auto sumqx = _mm_mul_ps(vw, _mm_mul_ps(vx, vq));
auto sumq2 = hsum_float_4(_mm_mul_ps(vw, _mm_mul_ps(vq, vq)));
sqx[i] = _mm_mul_ps(_mm_set1_ps(sumq2 > 0 ? 1/sumq2 : 0), _mm_mul_ps(sumqx, sumqx));
}
auto score = hsum_float_4x4(sqx);
auto mask = _mm_cmp_ps(score, vbest, _CMP_GT_OQ);
best_index = _mm_or_si128(_mm_and_si128(_mm_castps_si128(mask), idx),
_mm_andnot_si128(_mm_castps_si128(mask), best_index));
vbest = _mm_max_ps(vbest, score);
}
_mm_store_ps(sx, vbest);
_mm_store_si128((__m128i *)index, best_index);
for (int i = 0; i < 4; ++i) {
if (sx[i] > best) { best = sx[i]; jbest = index[i]; }
}
if (jbest < 0) {
fprintf(stderr, "Oops: jbest = %d for cluster %d with %d points\n", jbest, jbest_cluster, int(points.size()));
GGML_ASSERT(false);
}
best_idx[l] = jbest;
}
}
#else
// TODO
std::memset(best_idx, 0, kNg*sizeof(int));
#endif
}
template <int block_size, int group_size, int num_bits, int num_clusters>
void QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::find_best_match(float d, const float * xb, const float * weight, int * best_idx) const {
if (!d) { if (!d) {
std::memset(best_idx, 0, kNg*sizeof(int)); std::memset(best_idx, 0, kNg*sizeof(int));
return; return;
@@ -3460,39 +3318,45 @@ void QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::find_best_ma
auto wl = weight + 8*l; auto wl = weight + 8*l;
auto vx = _mm256_mul_ps(vid, _mm256_loadu_ps(xl)); auto vx = _mm256_mul_ps(vid, _mm256_loadu_ps(xl));
auto vw = _mm256_loadu_ps(wl); auto vw = _mm256_loadu_ps(wl);
auto vbest = _mm256_set1_ps(INFINITY); int jbest = -1;
auto best_index = _mm256_set1_epi32(-1); if (kGroupSize == 8 && ncluster == 256) {
float best = INFINITY; int jbest = -1; _mm256_store_ps(sx, vx);
auto idx = add_idx; uint8_t u = 0;
for (int j = 0; j < ncluster; j += 8) { for (int j = 0; j < 8; ++j) if (sx[j] > m_mid[j]) u |= (1 << j);
auto idx = _mm256_add_epi32(_mm256_set1_epi32(j), add_idx); jbest = u;
for (int i = 0; i < 8; ++i) { } else {
auto vq = _mm256_loadu_ps(m_clusters.data() + kGroupSize*(j+i)); auto vbest = _mm256_set1_ps(INFINITY);
auto vdiff = _mm256_sub_ps(vq, vx); auto best_index = _mm256_set1_epi32(-1);
sqx[i] = _mm256_mul_ps(vw, _mm256_mul_ps(vdiff, vdiff)); float best = INFINITY;
auto idx = add_idx;
for (int j = 0; j < ncluster; j += 8) {
for (int i = 0; i < 8; ++i) {
auto vq = _mm256_loadu_ps(m_clusters.data() + kGroupSize*(j+i));
auto vdiff = _mm256_sub_ps(vq, vx);
sqx[i] = _mm256_mul_ps(vw, _mm256_mul_ps(vdiff, vdiff));
}
auto score = hsum_float_8x8(sqx);
auto mask = _mm256_cmp_ps(score, vbest, _CMP_LT_OQ);
best_index = _mm256_or_si256(_mm256_and_si256(_mm256_castps_si256(mask), idx),
_mm256_andnot_si256(_mm256_castps_si256(mask), best_index));
vbest = _mm256_min_ps(vbest, score);
idx = _mm256_add_epi32(idx, add8);
}
_mm256_store_ps(sx, vbest);
_mm256_store_si256((__m256i *)index, best_index);
for (int i = 0; i < 8; ++i) {
if (sx[i] < best) { best = sx[i]; jbest = index[i]; }
} }
auto score = hsum_float_8x8(sqx);
auto mask = _mm256_cmp_ps(score, vbest, _CMP_LT_OQ);
best_index = _mm256_or_si256(_mm256_and_si256(_mm256_castps_si256(mask), idx),
_mm256_andnot_si256(_mm256_castps_si256(mask), best_index));
vbest = _mm256_min_ps(vbest, score);
idx = _mm256_add_epi32(idx, add8);
}
_mm256_store_ps(sx, vbest);
_mm256_store_si256((__m256i *)index, best_index);
for (int i = 0; i < 8; ++i) {
if (sx[i] < best) { best = sx[i]; jbest = index[i]; }
} }
auto& points = m_in_cluster[jbest]; auto& points = m_in_cluster[jbest];
auto& values = m_c_values[jbest]; auto& values = m_c_values[jbest];
GGML_ASSERT(!points.empty() && points.size()%8 == 0); GGML_ASSERT(!points.empty() && points.size()%8 == 0);
int jbest_cluster = jbest; int jbest_cluster = jbest;
vbest = _mm256_set1_ps(INFINITY); auto vbest = _mm256_set1_ps(INFINITY);
best_index = _mm256_set1_epi32(-1); auto best_index = _mm256_set1_epi32(-1);
best = INFINITY; jbest = -1; auto best = INFINITY; jbest = -1;
idx = add_idx; auto idx = add_idx;
for (int j = 0; j < int(points.size()); j += 8) { for (int j = 0; j < int(points.size()); j += 8) {
//auto idx = _mm256_add_epi32(_mm256_set1_epi32(j), add_idx);
for (int i = 0; i < 8; ++i) { for (int i = 0; i < 8; ++i) {
auto vq = _mm256_loadu_ps(values.data() + kGroupSize*(j+i)); auto vq = _mm256_loadu_ps(values.data() + kGroupSize*(j+i));
auto vdiff = _mm256_sub_ps(vq, vx); auto vdiff = _mm256_sub_ps(vq, vx);
@@ -3601,19 +3465,18 @@ void QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::find_best_ma
#endif #endif
} }
template <int block_size, int group_size, int num_bits, int num_clusters> template <int block_size, int group_size, int num_bits>
std::vector<std::vector<int>> QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::finalize_clusters( std::vector<std::vector<int>> QuantizerIQKT<block_size, group_size, num_bits>::finalize_clusters(int num_neighbours,
const std::vector<float>& values, const std::vector<float>& clusters, std::vector<std::vector<float>>& c_values) { const std::vector<float>& values, const std::vector<float>& clusters, std::vector<std::vector<float>>& c_values) {
constexpr int kNbest = 5;
int ncluster = clusters.size()/kGroupSize; int ncluster = clusters.size()/kGroupSize;
GGML_ASSERT(ncluster%8 == 0); GGML_ASSERT(ncluster%8 == 0);
std::vector<std::vector<int>> p_in_cluster(ncluster); std::vector<std::vector<int>> p_in_cluster(ncluster);
std::vector<int> which_cluster(kNbest*kNumVal); std::vector<int> which_cluster(num_neighbours*kNumVal);
int ibest[kNbest]; std::vector<int> ibest(num_neighbours);
float best[kNbest] = {INFINITY, INFINITY, INFINITY, INFINITY}; std::vector<float> best(num_neighbours);
for (int ip = 0; ip < kNumVal; ++ip) { for (int ip = 0; ip < kNumVal; ++ip) {
auto vp = values.data() + ip*kGroupSize; auto vp = values.data() + ip*kGroupSize;
for (int j = 0; j < kNbest; ++j) { for (int j = 0; j < num_neighbours; ++j) {
best[j] = INFINITY; ibest[j] = -1; best[j] = INFINITY; ibest[j] = -1;
} }
for (int ic = 0; ic < ncluster; ++ic) { for (int ic = 0; ic < ncluster; ++ic) {
@@ -3622,9 +3485,9 @@ std::vector<std::vector<int>> QuantizerIQKT<block_size, group_size, num_bits, nu
for (int k = 0; k < kGroupSize; ++k) { for (int k = 0; k < kGroupSize; ++k) {
float d = vp[k] - vc[k]; dist2 += d*d; float d = vp[k] - vc[k]; dist2 += d*d;
} }
for (int j = 0; j < kNbest; ++j) { for (int j = 0; j < num_neighbours; ++j) {
if (dist2 < best[j]) { if (dist2 < best[j]) {
for (int k = kNbest-1; k > j; --k) { for (int k = num_neighbours-1; k > j; --k) {
best[k] = best[k-1]; ibest[k] = ibest[k-1]; best[k] = best[k-1]; ibest[k] = ibest[k-1];
} }
best[j] = dist2; ibest[j] = ic; best[j] = dist2; ibest[j] = ic;
@@ -3632,14 +3495,14 @@ std::vector<std::vector<int>> QuantizerIQKT<block_size, group_size, num_bits, nu
} }
} }
} }
for (int j = 0; j < kNbest; ++j) { for (int j = 0; j < num_neighbours; ++j) {
if (ibest[j] < 0) { if (ibest[j] < 0) {
printf("Oops: ibest[%d] = %d\n", j, ibest[j]); printf("Oops: ibest[%d] = %d\n", j, ibest[j]);
} }
GGML_ASSERT(ibest[j] >= 0); GGML_ASSERT(ibest[j] >= 0);
p_in_cluster[ibest[j]].push_back(ip); p_in_cluster[ibest[j]].push_back(ip);
} }
std::memcpy(which_cluster.data() + kNbest*ip, ibest, kNbest*sizeof(int)); std::memcpy(which_cluster.data() + num_neighbours*ip, ibest.data(), num_neighbours*sizeof(int));
} }
std::vector<std::pair<float, int>> extra; std::vector<std::pair<float, int>> extra;
extra.reserve(kNumVal); extra.reserve(kNumVal);
@@ -3650,8 +3513,8 @@ std::vector<std::vector<int>> QuantizerIQKT<block_size, group_size, num_bits, nu
auto vc = clusters.data() + ic*kGroupSize; auto vc = clusters.data() + ic*kGroupSize;
for (int ip = 0; ip < kNumVal; ++ip) { for (int ip = 0; ip < kNumVal; ++ip) {
bool can_add = true; bool can_add = true;
for (int j = 0; j < kNbest; ++j) { for (int j = 0; j < num_neighbours; ++j) {
if (which_cluster[kNbest*ip+j] == ic) { can_add = false; break; } if (which_cluster[num_neighbours*ip+j] == ic) { can_add = false; break; }
} }
if (!can_add) continue; if (!can_add) continue;
auto vp = values.data() + ip*kGroupSize; auto vp = values.data() + ip*kGroupSize;
@@ -3690,8 +3553,8 @@ std::vector<std::vector<int>> QuantizerIQKT<block_size, group_size, num_bits, nu
return p_in_cluster; return p_in_cluster;
} }
template <int block_size, int group_size, int num_bits, int num_clusters> template <int block_size, int group_size, int num_bits>
std::vector<float> QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::cluster_points(const std::vector<float>& points, int ncluster, int niter) { std::vector<float> QuantizerIQKT<block_size, group_size, num_bits>::cluster_points(const std::vector<float>& points, int ncluster, int niter, float * mid) {
constexpr int ndim = kGroupSize; constexpr int ndim = kGroupSize;
GGML_ASSERT(points.size() % ndim == 0); GGML_ASSERT(points.size() % ndim == 0);
int npoint = points.size() / ndim; int npoint = points.size() / ndim;
@@ -3707,15 +3570,35 @@ std::vector<float> QuantizerIQKT<block_size, group_size, num_bits, num_clusters>
} }
} }
if (kVerbose) printf("%s (ndim = %d, npoint = %d): Fo = %g\n", __func__, ndim, npoint, Fo/points.size()); if (kVerbose) printf("%s (ndim = %d, npoint = %d): Fo = %g\n", __func__, ndim, npoint, Fo/points.size());
for (int k = 0; k < ndim; ++k) mid[k] = 0.5f*(range[k].first + range[k].second);
std::vector<float> sump(ncluster*ndim);
std::vector<int> counts(ncluster);
std::vector<float> result(ncluster*ndim);
if (group_size == 8 && ncluster == 256) {
std::memset(sump.data(), 0, sump.size()*sizeof(float));
std::memset(counts.data(), 0, counts.size()*sizeof(int));
for (int ip = 0; ip < npoint; ++ip) {
auto vp = points.data() + ndim*ip;
uint8_t u = 0;
for (int k = 0; k < ndim; ++k) if (vp[k] > mid[k]) u |= (1 << k);
++counts[u];
for (int k = 0; k < ndim; ++k) sump[ndim*u + k] += vp[k];
}
for (int ic = 0; ic < ncluster; ++ic) {
if (!counts[ic]) {
printf("%s: Oops. Cluster %d has no points\n", __func__, ic);
GGML_ABORT("fatal error");
}
for (int k = 0; k < ndim; ++k) result[ic*ndim + k] = sump[ic*ndim + k]/counts[ic];
}
return result;
}
std::mt19937 rndm(1234); std::mt19937 rndm(1234);
float scale = 1.f/4294967296.f; float scale = 1.f/4294967296.f;
std::vector<float> result(ncluster*ndim);
for (int i = 0; i < ncluster; ++i) { for (int i = 0; i < ncluster; ++i) {
auto v = result.data() + i*ndim; auto v = result.data() + i*ndim;
for (int k = 0; k < ndim; ++k) v[k] = range[k].first + (range[k].second - range[k].first)*scale*rndm(); for (int k = 0; k < ndim; ++k) v[k] = range[k].first + (range[k].second - range[k].first)*scale*rndm();
} }
std::vector<float> sump(ncluster*ndim);
std::vector<int> counts(ncluster);
std::vector<int> which_cluster(npoint, -1); std::vector<int> which_cluster(npoint, -1);
double Flast = Fo; double Flast = Fo;
for (int iter = 0; iter < niter; ++iter) { for (int iter = 0; iter < niter; ++iter) {
@@ -3758,13 +3641,14 @@ std::vector<float> QuantizerIQKT<block_size, group_size, num_bits, num_clusters>
return result; return result;
} }
using QuantizerIQ2KT = QuantizerIQKT<32, 8, 16, 128>; using QuantizerIQ2KT = QuantizerIQKT<32, 8, 16>;
const QuantizerIQ2KT& iq2kt_quantizer() { const QuantizerIQ2KT& iq2kt_quantizer() {
static std::mutex mutex; static std::mutex mutex;
static std::unique_ptr<QuantizerIQ2KT> quantizer;
std::lock_guard<std::mutex> lock(mutex); std::lock_guard<std::mutex> lock(mutex);
static QuantizerIQ2KT quantizer; if (!quantizer) quantizer = std::make_unique<QuantizerIQ2KT>(256, 8);
return quantizer; return *quantizer;
} }
void quantize_row_iq2_kt_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales, float * all_weights) { void quantize_row_iq2_kt_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales, float * all_weights) {
@@ -3970,12 +3854,13 @@ void vec_dot_iq2_kt_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx
namespace { namespace {
using QuantizerIQ3KT = QuantizerIQKT<32, 4, 12, 64>; using QuantizerIQ3KT = QuantizerIQKT<32, 4, 12>;
const QuantizerIQ3KT& iq3kt_quantizer() { const QuantizerIQ3KT& iq3kt_quantizer() {
static std::mutex mutex; static std::mutex mutex;
std::lock_guard<std::mutex> lock(mutex); std::lock_guard<std::mutex> lock(mutex);
static QuantizerIQ3KT quantizer; static std::unique_ptr<QuantizerIQ3KT> quantizer;
return quantizer; if (!quantizer) quantizer = std::make_unique<QuantizerIQ3KT>(64, 5);
return *quantizer;
} }
void quantize_row_iq3_kt_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales) { void quantize_row_iq3_kt_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales) {
@@ -4207,13 +4092,14 @@ void vec_dot_iq3_kt_q8_k(int n, float * s, size_t bs, const void * vx, size_t bx
namespace{ namespace{
using QuantizerIQ4KT = QuantizerIQKT<64, 4, 16, 512>; using QuantizerIQ4KT = QuantizerIQKT<64, 4, 16>;
const QuantizerIQ4KT& iq4kt_quantizer() { const QuantizerIQ4KT& iq4kt_quantizer() {
static std::mutex mutex; static std::mutex mutex;
std::lock_guard<std::mutex> lock(mutex); std::lock_guard<std::mutex> lock(mutex);
static QuantizerIQ4KT quantizer; static std::unique_ptr<QuantizerIQ4KT> quantizer;
return quantizer; if (!quantizer) quantizer = std::make_unique<QuantizerIQ4KT>(512, 5);
return *quantizer;
} }
void quantize_row_iq4_kt_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales, float * all_weights) { void quantize_row_iq4_kt_impl(const float * x, void * vy, int n_per_row, const float * quant_weights, float * all_scales, float * all_weights) {