ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-04-30 19:31:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

iq2_kt: SOTA

Browse Source 
We arrive at
PPL(LLaMA-3.1-8B-Instruct, 8192) = 9.1642
PPL(LLaMA-2-7B,            4096) = 6.3920
This commit is contained in:
Iwan Kawrakow
2024-11-11 09:30:22 +02:00
parent 4608f0cc6d
commit 47b28c1e92
1 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
ggml/src/iqk/iqk_quantize.cpp
View File

@@ -3500,10 +3500,11 @@ void QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::find_best_ma
float best = INFINITY; int jbest = -1; float best = INFINITY; int jbest = -1;
auto idx = add_idx; auto idx = add_idx;
for (int j = 0; j < ncluster; j += 8) { for (int j = 0; j < ncluster; j += 8) {
//auto idx = _mm256_add_epi32(_mm256_set1_epi32(j), add_idx);
for (int i = 0; i < 4; ++i) { for (int i = 0; i < 4; ++i) {
auto vq = _mm256_loadu_ps(m_clusters.data() + kGroupSize*(j+2*i)); auto vq = _mm256_loadu_ps(m_clusters.data() + kGroupSize*(j+2*i));
auto vdiff = _mm256_sub_ps(vq, vx_p); auto vdiff = _mm256_sub_ps(vq, vx_p);
//vdiff = _mm256_mul_ps(vdiff, vdiff);
//sqx[i] = _mm256_mul_ps(vw, _mm256_mul_ps(vdiff, vdiff));
vdiff = _mm256_andnot_ps(sign_bit, vdiff); vdiff = _mm256_andnot_ps(sign_bit, vdiff);
sqx[i] = _mm256_mul_ps(vw, _mm256_mul_ps(vdiff, _mm256_mul_ps(vdiff, vdiff))); sqx[i] = _mm256_mul_ps(vw, _mm256_mul_ps(vdiff, _mm256_mul_ps(vdiff, vdiff)));
} }
@@ -3528,10 +3529,11 @@ void QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::find_best_ma
best = INFINITY; jbest = -1; best = INFINITY; jbest = -1;
idx = add_idx; idx = add_idx;
for (int j = 0; j < int(points.size()); j += 8) { for (int j = 0; j < int(points.size()); j += 8) {
//auto idx = _mm256_add_epi32(_mm256_set1_epi32(j), add_idx);
for (int i = 0; i < 4; ++i) { for (int i = 0; i < 4; ++i) {
auto vq = _mm256_loadu_ps(values.data() + kGroupSize*(j+2*i)); auto vq = _mm256_loadu_ps(values.data() + kGroupSize*(j+2*i));
auto vdiff = _mm256_sub_ps(vq, vx_p); auto vdiff = _mm256_sub_ps(vq, vx_p);
//vdiff = _mm256_mul_ps(vdiff, vdiff);
//sqx[i] = _mm256_mul_ps(vw, _mm256_mul_ps(vdiff, vdiff));
vdiff = _mm256_andnot_ps(sign_bit, vdiff); vdiff = _mm256_andnot_ps(sign_bit, vdiff);
sqx[i] = _mm256_mul_ps(vw, _mm256_mul_ps(vdiff, _mm256_mul_ps(vdiff, vdiff))); sqx[i] = _mm256_mul_ps(vw, _mm256_mul_ps(vdiff, _mm256_mul_ps(vdiff, vdiff)));
} }
@@ -3563,7 +3565,7 @@ void QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::find_best_ma
template <int block_size, int group_size, int num_bits, int num_clusters> template <int block_size, int group_size, int num_bits, int num_clusters>
std::vector<std::vector<int>> QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::finalize_clusters( std::vector<std::vector<int>> QuantizerIQKT<block_size, group_size, num_bits, num_clusters>::finalize_clusters(
const std::vector<float>& values, const std::vector<float>& clusters, std::vector<std::vector<float>>& c_values) { const std::vector<float>& values, const std::vector<float>& clusters, std::vector<std::vector<float>>& c_values) {
constexpr int kNbest = 4; constexpr int kNbest = 5;
int ncluster = clusters.size()/kGroupSize; int ncluster = clusters.size()/kGroupSize;
GGML_ASSERT(ncluster%8 == 0); GGML_ASSERT(ncluster%8 == 0);
std::vector<std::vector<int>> p_in_cluster(ncluster); std::vector<std::vector<int>> p_in_cluster(ncluster);