From 31988c7eea7446cc3f502c3fbe7562281c3e2aaf Mon Sep 17 00:00:00 2001 From: Iwan Kawrakow Date: Fri, 23 May 2025 13:32:50 +0300 Subject: [PATCH] Cleanup --- ggml/src/iqk/iqk_gemm_ktquants.cpp | 80 ------------------------------ 1 file changed, 80 deletions(-) diff --git a/ggml/src/iqk/iqk_gemm_ktquants.cpp b/ggml/src/iqk/iqk_gemm_ktquants.cpp index 1045f34d..b7abc2e1 100644 --- a/ggml/src/iqk/iqk_gemm_ktquants.cpp +++ b/ggml/src/iqk/iqk_gemm_ktquants.cpp @@ -189,27 +189,6 @@ static inline __m256 abs_ps(__m256 vals) { return _mm256_andnot_ps(sign_bit, vals); } -// Negates 32-bit float lanes of an 8x32-bit vector -// based on 8x8-bit condition var. For float lane i, if byte i of -// `condition` is nonzero, the float will be negated. -static inline __m256 conditional_negate_ps(__m256 vals, __m128i condition_bytes) { - // Make `should_negate_byte_mask` where byte i == 0xFF if byte i in condition_bytes is zero, - // else 0x00 (upper bytes are meaningless) - __m128i zeros = _mm_setzero_si128(); - __m128i is_zero_byte_mask = _mm_cmpeq_epi8(condition_bytes, zeros); - __m128i should_negate_byte_mask = _mm_cmpeq_epi8(is_zero_byte_mask, zeros); - // Widen lower 8x8 bits of `should_negate_byte_mask` to 8x32 bits by padding zeros - // expanded_mask_epi32[j] will be 0x000000FF if vals[j] should be negated, zero otherwise - __m256i expanded_mask_epi32 = _mm256_cvtepu8_epi32(should_negate_byte_mask); - // Same as above but with all 32 bits of lane j set if vals[j] should be negated (use to make XOR mask) - __m256i full_dword_negate_mask = _mm256_cmpgt_epi32(expanded_mask_epi32, _mm256_setzero_si256()); - // Negate via XOR on sign bits of each 32-bit float - __m256i sign_bit_pattern = _mm256_set1_epi32(0x80000000); // MSB set for a 32-bit value - __m256i xor_mask_epi32 = _mm256_and_si256(full_dword_negate_mask, sign_bit_pattern); - __m256 xor_mask_ps = _mm256_castsi256_ps(xor_mask_epi32); - return _mm256_xor_ps(vals, xor_mask_ps); -} - template static void mul_mat_iq3_kt_F32_T(int n, const void * vx, size_t bx, const DataInfo& info, int nrc_x) { assert(n%QK_K == 0); @@ -220,9 +199,6 @@ static void mul_mat_iq3_kt_F32_T(int n, const void * vx, size_t bx, const DataIn union { __m256 vec; float val[8]; } s_helper; auto shifts = _mm_set_epi32(0, 0, 4, 0); - //auto sign_mask1 = _mm256_set1_epi32(0x01); - //auto sign_mask2 = _mm256_set1_epi32(0x10); - //auto sign_bit = _mm256_set1_ps(-0.0f); __m256i all_signs[4]; auto mask1 = _mm256_set1_epi32(0x01); @@ -246,25 +222,16 @@ static void mul_mat_iq3_kt_F32_T(int n, const void * vx, size_t bx, const DataIn s8 = _mm_and_si128(_mm_srlv_epi32(s8, shifts), _mm_set1_epi8(0xf)); auto s32 = _mm256_cvtepi8_epi32(s8); s_helper.vec = _mm256_cvtepi32_ps(s32); - //auto mask1 = _mm_set1_epi8(1); - //auto mask2 = _mm_slli_epi16(mask1, 4); for (int j = 0; j < 4; ++j) all_signs[j] = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)(qh + 8*j))); for (int ib = 0; ib < 4; ++ib) { - //auto sign_bits = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)(x[i].qh + 8*ib))); auto scale1 = _mm256_set1_ps(s_helper.val[ib+0]); auto scale2 = _mm256_set1_ps(s_helper.val[ib+4]); - //uint64_t mask1 = 0x0101010101010101 << ib; //(j/32); - //uint64_t mask2 = mask1 << 4; for (int j = 0; j < 4; ++j) { uint32_t val1 = ql[4*ib+j ] + 4096; uint32_t val2 = ql[4*ib+j+16] + 4096; auto sign1 = _mm256_and_si256(_mm256_cmpeq_epi32(_mm256_and_si256(all_signs[j], mask1), mask1), _mm256_set1_epi32(0x80000000)); auto sign2 = _mm256_and_si256(_mm256_cmpeq_epi32(_mm256_and_si256(all_signs[j], mask2), mask2), _mm256_set1_epi32(0x80000000)); all_signs[j] = _mm256_srli_epi32(all_signs[j], 1); - //auto signs = _mm_loadl_epi64((const __m128i *)(qh + j)); - //auto sign1 = _mm_and_si128(signs, mask1); - //auto sign2 = _mm_and_si128(signs, mask2); - //const uint64_t signs = *((const uint64_t *)(qh + j)); auto x_val1 = abs_ps(trellis_gen8(trellis.next8(val1))); auto x_val2 = abs_ps(trellis_gen8(trellis.next8(val2))); x_val1 = _mm256_mul_ps(scale1, _mm256_xor_ps(x_val1, _mm256_castsi256_ps(sign1))); @@ -274,54 +241,7 @@ static void mul_mat_iq3_kt_F32_T(int n, const void * vx, size_t bx, const DataIn accd[iy] = _mm256_fmadd_ps(_mm256_load_ps(y[iy] + i*QK_K+32*ib+8*j+128), x_val2, accd[iy]); } } - //mask1 = _mm_slli_epi16(mask1, 1); - //mask2 = _mm_slli_epi16(mask2, 1); - //for (int j = 0; j < 4; ++j) { - // //auto signs1 = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_and_si256(sign_bits, sign_mask1), 27)); - // //auto signs2 = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_and_si256(sign_bits, sign_mask2), 23)); - // //sign_bits = _mm256_srli_epi32(sign_bits, 1); - // auto smask1 = _mm256_cmpeq_epi32(_mm256_and_si256(sign_bits, sign_mask1), sign_mask1); - // auto smask2 = _mm256_cmpeq_epi32(_mm256_and_si256(sign_bits, sign_mask2), sign_mask2); - // sign_bits = _mm256_srli_epi32(sign_bits, 1); - // auto signs1 = _mm256_and_ps(_mm256_castsi256_ps(smask1), sign_bit); - // auto signs2 = _mm256_and_ps(_mm256_castsi256_ps(smask2), sign_bit); - // auto a_val1 = _mm256_andnot_ps(sign_bit, trellis_gen8(trellis.next8(ql[4*ib+j+ 0]+4096))); - // auto a_val2 = _mm256_andnot_ps(sign_bit, trellis_gen8(trellis.next8(ql[4*ib+j+16]+4096))); - // auto x_val1 = _mm256_mul_ps(scale1, _mm256_xor_ps(a_val1, signs1)); - // auto x_val2 = _mm256_mul_ps(scale2, _mm256_xor_ps(a_val2, signs2)); - // for (int iy = 0; iy < nrc_y; ++iy) { - // accd[iy] = _mm256_fmadd_ps(_mm256_loadu_ps(y[iy] + i*QK_K + 32*ib + 8*j + 0), x_val1, accd[iy]); - // accd[iy] = _mm256_fmadd_ps(_mm256_loadu_ps(y[iy] + i*QK_K + 32*ib + 8*j + 128), x_val2, accd[iy]); - // } - //} } - //for (int j = 0; j < 128; j+=8) { - // uint64_t mask1 = 0x0101010101010101 << (j/32); - // uint64_t mask2 = mask1 << 4; - // uint32_t val1 = ql[j/8] + 4096; - // uint32_t val2 = ql[j/8+16] + 4096; - // const uint64_t signs = *((const uint64_t *)(qh + (j%32))); - // const float x_scale1 = (x[i].scales[j/32] & 0xf); - // const float x_scale2 = (x[i].scales[j/32] >> 4); - // const __m256 x_val1 = abs_ps(trellis_gen8(trellis.next8(val1))); - // const __m256 x_val2 = abs_ps(trellis_gen8(trellis.next8(val2))); - // for (int iy = 0; iy < nrc_y; ++iy) { - // accd[iy] = _mm256_fmadd_ps( - // conditional_negate_ps( - // _mm256_load_ps(y[iy] + i*QK_K+j), signs & mask1 - // ), - // _mm256_mul_ps(_mm256_set1_ps(x_scale1), x_val1), - // accd[iy] - // ); - // accd[iy] = _mm256_fmadd_ps( - // conditional_negate_ps( - // _mm256_load_ps(y[iy] + i*QK_K+j+128), signs & mask2 - // ), - // _mm256_mul_ps(_mm256_set1_ps(x_scale2), x_val2), - // accd[iy] - // ); - // } - //} } for (int iy = 0; iy < nrc_y; ++iy) {