From e06fa8b0d2b4363a4e412fb1f15903ec62fbafc9 Mon Sep 17 00:00:00 2001 From: varshav2 Date: Tue, 17 Sep 2024 22:14:27 +0530 Subject: [PATCH] Add Transpose Kernel for A matrix in F32F32f32Of32 - Implemented the AVX512 packA kernel for col major inputs in F32 API - Removed the work arounds for n = 1, mtag_a = PACK case, where the execution was being directed to GEMM instead of GEMV. Change-Id: I6fb700d96069213a762e8a83a209c5388a91050f --- addon/aocl_gemm/aocl_gemm.h | 1 + addon/aocl_gemm/aocl_gemm_f32f32f32of32.c | 1 - addon/aocl_gemm/config/lpgemm_config.c | 1 + .../frame/f32f32f32/lpgemm_f32f32f32.c | 83 +- .../kernels/f32f32f32/lpgemm_pack_f32.h | 50 ++ .../f32f32f32/lpgemm_pack_a_f32_amd512vnni.c | 762 ++++++++++++++++++ 6 files changed, 857 insertions(+), 41 deletions(-) create mode 100644 addon/aocl_gemm/kernels/f32f32f32/lpgemm_pack_f32.h create mode 100644 kernels/zen4/lpgemm/f32f32f32/lpgemm_pack_a_f32_amd512vnni.c diff --git a/addon/aocl_gemm/aocl_gemm.h b/addon/aocl_gemm/aocl_gemm.h index 9a8030b4a..070f05bc7 100644 --- a/addon/aocl_gemm/aocl_gemm.h +++ b/addon/aocl_gemm/aocl_gemm.h @@ -53,6 +53,7 @@ #include "lpgemm_packa_s8.h" #include "lpgemm_packb_s8.h" #include "lpgemm_packb_s8s16.h" +#include "lpgemm_pack_f32.h" #include "lpgemm_jit_typedefs.h" #ifdef LPGEMM_BF16_JIT #include "lpgemm_jit_c_connector.h" diff --git a/addon/aocl_gemm/aocl_gemm_f32f32f32of32.c b/addon/aocl_gemm/aocl_gemm_f32f32f32of32.c index d759ad6e0..e3db6e386 100644 --- a/addon/aocl_gemm/aocl_gemm_f32f32f32of32.c +++ b/addon/aocl_gemm/aocl_gemm_f32f32f32of32.c @@ -168,7 +168,6 @@ AOCL_GEMM_MATMUL(float,float,float,float,f32f32f32of32) bli_pba_rntm_set_pba( &rntm_g ); lpgemm_cntx_t* lcntx_g = lpgemm_get_global_cntx_obj( F32F32F32OF32 ); - #ifdef BLIS_ENABLE_OPENMP // The lpgemm_cntx_t argument will be NULL for f32 since it still uses // BLIS cntx_t internally. Its a workaround for now and will be replaced diff --git a/addon/aocl_gemm/config/lpgemm_config.c b/addon/aocl_gemm/config/lpgemm_config.c index e3a4c7b6e..3fee7d2cb 100644 --- a/addon/aocl_gemm/config/lpgemm_config.c +++ b/addon/aocl_gemm/config/lpgemm_config.c @@ -45,6 +45,7 @@ #include "lpgemm_packa_s8.h" #include "lpgemm_packb_s8.h" #include "lpgemm_packb_s8s16.h" +#include "lpgemm_pack_f32.h" static lpgemm_cntx_t global_cntx_t_list[AOCL_OPERATION_TYPE_LEN] \ __attribute__((aligned(64))); //Only one op type supported now. diff --git a/addon/aocl_gemm/frame/f32f32f32/lpgemm_f32f32f32.c b/addon/aocl_gemm/frame/f32f32f32/lpgemm_f32f32f32.c index bdac01c02..c94a8d80d 100644 --- a/addon/aocl_gemm/frame/f32f32f32/lpgemm_f32f32f32.c +++ b/addon/aocl_gemm/frame/f32f32f32/lpgemm_f32f32f32.c @@ -38,6 +38,7 @@ #include "lpgemm_utils.h" #include "lpgemm_thrinfo_utils.h" #include "lpgemm_kernels.h" +#include "lpgemm_pack_f32.h" // Kernel function prototypes typedef void (*lpgemm_rowvar_f32) @@ -96,7 +97,6 @@ LPGEMV(float, float, float, f32f32f32of32) const float* a_use = (float*)a; inc_t rs_a_use = rs_a; inc_t cs_a_use = cs_a; - inc_t ps_a_use; float* b_use = (float*)b; inc_t rs_b_use = rs_b; @@ -129,15 +129,13 @@ LPGEMV(float, float, float, f32f32f32of32) thrinfo_t thread_jc; thrinfo_t thread_ic; lpgemm_gen_thrinfo(thread, &thread_jc, &thread_ic); - + if(n == 1) { float* pack_b_buffer_f32f32f32of32; - //TODO: AVX2 support need to be added // Increased MR from 6 to 16 to make use of 32 ZMM registers dim_t MR = 16; - // Pack B matrix if rs_b > 1 if( ( mtag_b == PACK ) && ( rs_b != 1 ) ) { @@ -175,7 +173,7 @@ LPGEMV(float, float, float, f32f32f32of32) c_use = c + ic * rs_c; post_ops_attr.post_op_c_i = ic; - if( mtag_a == PACK ) + if( mtag_a == PACK && cs_a != 1 ) { mem_a_size_req = sizeof(float) * mc0 * k; lpgemm_alloc_mem_panel @@ -185,19 +183,13 @@ LPGEMV(float, float, float, f32f32f32of32) ); pack_a_buffer_f32f32f32of32 = ( float* )bli_mem_buffer( &mem_a ); - rs_a_use = 1; - cs_a_use = MR; - ps_a_use = MR * k; - - lpgemm_pack_a_f32f32f32of32 - ( - a_use, + packa_mr16_f32f32f32of32_col_major + ( pack_a_buffer_f32f32f32of32, + a_use, rs_a, cs_a, mc0, k, - rs_a, cs_a, ps_a_use, MR, - cntx - ); - + &rs_a_use, &cs_a_use + ); a_use = pack_a_buffer_f32f32f32of32; } @@ -224,14 +216,38 @@ LPGEMV(float, float, float, f32f32f32of32) } } else - { + { // Compute the JC loop thread range for the current thread. dim_t jc_start, jc_end; thread_jc.n_way = ( thread_jc.n_way == 1 ) ? ( thread->n_threads ) : ( thread_jc.n_way ); thread_jc.work_id = thread->tid; bli_thread_range_sub(&thread_jc, n, NR, FALSE, &jc_start, &jc_end); - + + if ( mtag_a == PACK ) + { + mem_a_size_req = sizeof( float ) * k; + + lpgemm_alloc_mem_panel + ( + mem_a_size_req, BLIS_BUFFER_FOR_GEN_USE, + &mem_a, rntm + ); + + pack_a_buffer_f32f32f32of32 = + ( float* ) bli_mem_buffer( &mem_a ); + + packa_mr16_f32f32f32of32_col_major + ( + pack_a_buffer_f32f32f32of32, + a_use, rs_a, cs_a, + 1, k, + &rs_a_use, &cs_a_use + ); + + a_use = pack_a_buffer_f32f32f32of32; + } + for (dim_t jc = jc_start; jc < jc_end; jc += NC) { dim_t nc0 = bli_min((jc_end - jc), NC); @@ -253,7 +269,7 @@ LPGEMV(float, float, float, f32f32f32of32) rs_b_use = NR; cs_b_use = 1; } - else if (mtag_b == PACK) + else if (mtag_b == PACK) { // nc0 needs to be a multiple of 16 since this gives maximum // vectorization. Packing B always results in buffers with width @@ -263,20 +279,20 @@ LPGEMV(float, float, float, f32f32f32of32) mem_b_size_req = sizeof( float ) * nc0_updated * k; n_sub_updated = nc0_updated; - + lpgemm_alloc_mem_panel ( mem_b_size_req, BLIS_BUFFER_FOR_B_PANEL, &mem_b, rntm ); - + pack_b_buffer_f32f32f32of32 = ( float* ) bli_mem_buffer( &mem_b ); - + for ( dim_t pc = 0; pc < k; pc += KC ) { dim_t kc0 = bli_min( ( k - pc ), KC ); - + // Set the strides for pack buffer. rs_b_use = NR; cs_b_use = 1; @@ -289,7 +305,7 @@ LPGEMV(float, float, float, f32f32f32of32) nc0 , kc0, rs_b, cs_b, ( NR * ps_b_use ), NR, cntx - ); + ); } b_use = pack_b_buffer_f32f32f32of32; } @@ -297,7 +313,7 @@ LPGEMV(float, float, float, f32f32f32of32) { b_use = (float*) b + jc * cs_b; } - + //update post-op pointer post_ops_attr.post_op_c_j = jc; @@ -305,7 +321,7 @@ LPGEMV(float, float, float, f32f32f32of32) lpgemv_m_one_f32f32f32of32 ( nc0, k, - a, rs_a, cs_a, mtag_a, + a_use, rs_a_use, cs_a_use, mtag_a, b_use, rs_b_use, cs_b_use, mtag_b, c_use, rs_c, cs_c, alpha, beta, @@ -336,10 +352,7 @@ LPGEMM_5LOOP(float, float, float, f32f32f32of32) #ifdef BLIS_KERNELS_ZEN4 // Handle using LPGEMV when m or/and n equal to 1 // The avx512 check will be removed when avx2 kernels added in future - //ToDo: with trasnsA row storage and transB column storage, the packed matrices will be in col stored row access - //which will give error in the computation. Hence, for now redirecting those cases to GEMM instead of GEMV to avoid the errors. - if ( ( ( m == 1 ) || ( n == 1 ) ) && (bli_cpuid_is_avx512_supported() == TRUE) && - ( mtag_a != PACK ) ) + if ( ( ( m == 1 ) || ( n == 1 ) ) && (bli_cpuid_is_avx512_supported() == TRUE) ) { lpgemv_rowvar_f32f32f32of32(m, n, k, a, rs_a, cs_a, mtag_a, @@ -355,16 +368,6 @@ LPGEMM_5LOOP(float, float, float, f32f32f32of32) return; } #endif - //ToDo: In case of transA with row storage, the padding will not be done if mtag_a is enabled by user. - //This would give a seg fault. Hence, adding the condition here so that this will be taken care. - if( ( n == 1 ) && ( mtag_a == PACK ) ) { - if(mtag_b == REORDERED) { - rs_b = 1; - cs_b = 1; - } - mtag_b = PACK; - } - // Query the global cntx. cntx_t* cntx = bli_gks_query_cntx(); diff --git a/addon/aocl_gemm/kernels/f32f32f32/lpgemm_pack_f32.h b/addon/aocl_gemm/kernels/f32f32f32/lpgemm_pack_f32.h new file mode 100644 index 000000000..3f799bba7 --- /dev/null +++ b/addon/aocl_gemm/kernels/f32f32f32/lpgemm_pack_f32.h @@ -0,0 +1,50 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2024, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ +#ifndef BLIS_GEMM_F32_PACKA +#define BLIS_GEMM_F32_PACKA + +void packa_mr16_f32f32f32of32_col_major + ( + float* pack_a_buffer, + const float* a, + const dim_t rs_a, + const dim_t cs_a, + const dim_t MC, + const dim_t KC, + dim_t* rs_p, + dim_t* cs_p + ); +#endif + + diff --git a/kernels/zen4/lpgemm/f32f32f32/lpgemm_pack_a_f32_amd512vnni.c b/kernels/zen4/lpgemm/f32f32f32/lpgemm_pack_a_f32_amd512vnni.c new file mode 100644 index 000000000..631bef66c --- /dev/null +++ b/kernels/zen4/lpgemm/f32f32f32/lpgemm_pack_a_f32_amd512vnni.c @@ -0,0 +1,762 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2024, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binarsy form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include +#include +#include "blis.h" + +#ifdef BLIS_ADDON_LPGEMM + +#define UNPACKLO_PS16 \ + b_reg[0] = _mm512_unpacklo_ps(a_reg[0], a_reg[1]); \ + b_reg[1] = _mm512_unpacklo_ps(a_reg[2], a_reg[3]); \ + b_reg[2] = _mm512_unpacklo_ps(a_reg[4], a_reg[5]); \ + b_reg[3] = _mm512_unpacklo_ps(a_reg[6], a_reg[7]); \ + b_reg[4] = _mm512_unpacklo_ps(a_reg[8], a_reg[9]); \ + b_reg[5] = _mm512_unpacklo_ps(a_reg[10], a_reg[11]); \ + b_reg[6] = _mm512_unpacklo_ps(a_reg[12], a_reg[13]); \ + b_reg[7] = _mm512_unpacklo_ps(a_reg[14], a_reg[15]); + +#define UNPACKHI_PS16 \ + b_reg[8] = _mm512_unpackhi_ps(a_reg[0], a_reg[1]); \ + b_reg[9] = _mm512_unpackhi_ps(a_reg[2], a_reg[3]); \ + b_reg[10] = _mm512_unpackhi_ps(a_reg[4], a_reg[5]); \ + b_reg[11] = _mm512_unpackhi_ps(a_reg[6], a_reg[7]); \ + b_reg[12] = _mm512_unpackhi_ps(a_reg[8], a_reg[9]); \ + b_reg[13] = _mm512_unpackhi_ps(a_reg[10], a_reg[11]); \ + b_reg[14] = _mm512_unpackhi_ps(a_reg[12], a_reg[13]); \ + b_reg[15] = _mm512_unpackhi_ps(a_reg[14], a_reg[15]); + +#define SHUFFLE_64x2 \ + a_reg[0] = _mm512_shuffle_ps(b_reg[0], b_reg[1], 0x44); \ + a_reg[1] = _mm512_shuffle_ps(b_reg[0], b_reg[1], 0xEE); \ + a_reg[2] = _mm512_shuffle_ps(b_reg[2], b_reg[3], 0x44); \ + a_reg[3] = _mm512_shuffle_ps(b_reg[2], b_reg[3], 0xEE); \ +\ + a_reg[4] = _mm512_shuffle_ps(b_reg[4], b_reg[5], 0x44); \ + a_reg[5] = _mm512_shuffle_ps(b_reg[4], b_reg[5], 0xEE); \ + a_reg[6] = _mm512_shuffle_ps(b_reg[6], b_reg[7], 0x44); \ + a_reg[7] = _mm512_shuffle_ps(b_reg[6], b_reg[7], 0xEE); \ +\ + a_reg[8] = _mm512_shuffle_ps(b_reg[8], b_reg[9], 0x44); \ + a_reg[9] = _mm512_shuffle_ps(b_reg[8], b_reg[9], 0xEE); \ + a_reg[10] = _mm512_shuffle_ps(b_reg[10], b_reg[11], 0x44); \ + a_reg[11] = _mm512_shuffle_ps(b_reg[10], b_reg[11], 0xEE); \ +\ + a_reg[12] = _mm512_shuffle_ps(b_reg[12], b_reg[13], 0x44); \ + a_reg[13] = _mm512_shuffle_ps(b_reg[12], b_reg[13], 0xEE); \ + a_reg[14] = _mm512_shuffle_ps(b_reg[14], b_reg[15], 0x44); \ + a_reg[15] = _mm512_shuffle_ps(b_reg[14], b_reg[15], 0xEE); + +#define MASKED_STORE_PS(mask) \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+0) * KC + kr ), mask, a_reg[0]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+1) * KC + kr ), mask, a_reg[1]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+2) * KC + kr ), mask, a_reg[2]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+3) * KC + kr ), mask, a_reg[3]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+4) * KC + kr ), mask, a_reg[4]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+5) * KC + kr ), mask, a_reg[5]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+6) * KC + kr ), mask, a_reg[6]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+7) * KC + kr ), mask, a_reg[7]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+8) * KC + kr ), mask, a_reg[8]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+9) * KC + kr ), mask, a_reg[9]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+10) * KC + kr ), mask, a_reg[10]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+11) * KC + kr ), mask, a_reg[11]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+12) * KC + kr ), mask, a_reg[12]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+13) * KC + kr ), mask, a_reg[13]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+14) * KC + kr ), mask, a_reg[14]); \ + _mm512_mask_storeu_ps((pack_a_buffer + (ic+15) * KC + kr ), mask, a_reg[15]); + +#define PERMUTE4x4( mask1, mask2 ) \ + b_reg[0] = _mm512_permutex2var_ps( a_reg[0], mask1, a_reg[2] ); \ + b_reg[1] = _mm512_permutex2var_ps( a_reg[1], mask1, a_reg[3] ); \ + b_reg[2] = _mm512_permutex2var_ps( a_reg[8], mask1, a_reg[10] ); \ + b_reg[3] = _mm512_permutex2var_ps( a_reg[9], mask1, a_reg[11] ); \ +\ + b_reg[4] = _mm512_permutex2var_ps( a_reg[4], mask1, a_reg[6]); \ + b_reg[5] = _mm512_permutex2var_ps( a_reg[5], mask1, a_reg[7]); \ + b_reg[6] = _mm512_permutex2var_ps( a_reg[12], mask1, a_reg[14]); \ + b_reg[7] = _mm512_permutex2var_ps( a_reg[13], mask1, a_reg[15]); \ +\ + b_reg[8] = _mm512_permutex2var_ps( a_reg[0], mask2, a_reg[2]); \ + b_reg[9] = _mm512_permutex2var_ps( a_reg[1], mask2, a_reg[3]); \ + b_reg[10] = _mm512_permutex2var_ps( a_reg[8], mask2, a_reg[10]); \ + b_reg[11] = _mm512_permutex2var_ps( a_reg[9], mask2, a_reg[11]); \ +\ + b_reg[12] = _mm512_permutex2var_ps( a_reg[4], mask2, a_reg[6]); \ + b_reg[13] = _mm512_permutex2var_ps( a_reg[5], mask2, a_reg[7]); \ + b_reg[14] = _mm512_permutex2var_ps( a_reg[12], mask2, a_reg[14]); \ + b_reg[15] = _mm512_permutex2var_ps( a_reg[13], mask2, a_reg[15]); + +#define PERMUTE8x8( mask3, mask4 ) \ + a_reg[0] = _mm512_permutex2var_ps( b_reg[0], mask3, b_reg[4]); \ + a_reg[1] = _mm512_permutex2var_ps( b_reg[1], mask3, b_reg[5]); \ + a_reg[2] = _mm512_permutex2var_ps( b_reg[2], mask3, b_reg[6]); \ + a_reg[3] = _mm512_permutex2var_ps( b_reg[3], mask3, b_reg[7]); \ +\ + a_reg[4] = _mm512_permutex2var_ps( b_reg[0], mask4, b_reg[4]); \ + a_reg[5] = _mm512_permutex2var_ps( b_reg[1], mask4, b_reg[5]); \ + a_reg[6] = _mm512_permutex2var_ps( b_reg[2], mask4, b_reg[6]); \ + a_reg[7] = _mm512_permutex2var_ps( b_reg[3], mask4, b_reg[7]); \ +\ + a_reg[8] = _mm512_permutex2var_ps( b_reg[8], mask3, b_reg[12]); \ + a_reg[9] = _mm512_permutex2var_ps( b_reg[9], mask3, b_reg[13]); \ + a_reg[10] = _mm512_permutex2var_ps( b_reg[10], mask3, b_reg[14]); \ + a_reg[11] = _mm512_permutex2var_ps( b_reg[11], mask3, b_reg[15]); \ +\ + a_reg[12] = _mm512_permutex2var_ps( b_reg[8], mask4, b_reg[12]); \ + a_reg[13] = _mm512_permutex2var_ps( b_reg[9], mask4, b_reg[13]); \ + a_reg[14] = _mm512_permutex2var_ps( b_reg[10], mask4, b_reg[14]); \ + a_reg[15] = _mm512_permutex2var_ps( b_reg[11], mask4, b_reg[15]); + +void packa_mr16_f32f32f32of32_col_major +( + float* pack_a_buffer, + const float* a, + const dim_t rs_a, + const dim_t cs_a, + const dim_t MC, + const dim_t KC, + dim_t* rs_p, + dim_t* cs_p +) +{ + dim_t MR = 16; + dim_t ic, kr; + dim_t m_left = MC % 4; + + __m512 a_reg[16], b_reg[16]; + + __m512i mask1 = _mm512_set_epi32( 0x17, 0x16, 0x15, 0x14, + 0x07, 0x06, 0x05, 0x04, + 0x13, 0x12, 0x11, 0x10, + 0x03, 0x02, 0x01, 0x00 ); + + __m512i mask2 = _mm512_set_epi32( 0x1F, 0x1E, 0x1D, 0x1C, + 0x0F, 0x0E, 0x0D, 0x0C, + 0x1B, 0x1A, 0x19, 0x18, + 0x0B, 0x0A, 0x9, 0x08 ); + + __m512i mask3 = _mm512_set_epi32( 0x17, 0x16, 0x15, 0x14, 0x13, 0x12, 0x11, 0x10, + 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00 ); + __m512i mask4 = _mm512_set_epi32( 0x1F, 0x1E, 0x1D, 0x1C, 0x1B, 0x1A, 0x19, 0x18, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 ); + + // These registers are set with zeroes to avoid compiler warnings + // To-DO: TO be removed when pack code is optimized for fringe cases. + a_reg[0] = _mm512_setzero_ps(); + a_reg[1] = _mm512_setzero_ps(); + a_reg[2] = _mm512_setzero_ps(); + a_reg[3] = _mm512_setzero_ps(); + a_reg[4] = _mm512_setzero_ps(); + a_reg[5] = _mm512_setzero_ps(); + a_reg[6] = _mm512_setzero_ps(); + a_reg[7] = _mm512_setzero_ps(); + a_reg[8] = _mm512_setzero_ps(); + a_reg[9] = _mm512_setzero_ps(); + a_reg[10] = _mm512_setzero_ps(); + a_reg[11] = _mm512_setzero_ps(); + a_reg[12] = _mm512_setzero_ps(); + a_reg[13] = _mm512_setzero_ps(); + a_reg[14] = _mm512_setzero_ps(); + a_reg[15] = _mm512_setzero_ps(); + + for( ic = 0; ( ic + MR - 1 ) < MC; ic += MR) + { + for( kr = 0; ( kr + 15 ) < KC; kr += 16) + { + a_reg[0] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ) ); + a_reg[1] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ) ); + a_reg[2] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) ) ); + a_reg[3] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) ) ); + a_reg[4] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 4 ) * cs_a ) ) ); + a_reg[5] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 5 ) * cs_a ) ) ); + a_reg[6] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 6 ) * cs_a ) ) ); + a_reg[7] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 7 ) * cs_a ) ) ); + a_reg[8] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 8 ) * cs_a ) ) ); + a_reg[9] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 9 ) * cs_a ) ) ); + a_reg[10] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 10 ) * cs_a ) ) ); + a_reg[11] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 11 ) * cs_a ) ) ); + a_reg[12] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 12 ) * cs_a ) ) ); + a_reg[13] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 13 ) * cs_a ) ) ); + a_reg[14] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 14 ) * cs_a ) ) ); + a_reg[15] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 15 ) * cs_a ) ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ); + PERMUTE8x8( mask3, mask4 ) + + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), a_reg[0] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), a_reg[1] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), a_reg[2] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), a_reg[3] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 4 ) * KC + kr ), a_reg[4] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 5 ) * KC + kr ), a_reg[5] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 6 ) * KC + kr ), a_reg[6] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 7 ) * KC + kr ), a_reg[7] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 8 ) * KC + kr ), a_reg[8] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 9 ) * KC + kr ), a_reg[9] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 10 ) * KC + kr ), a_reg[10] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 11 ) * KC + kr ), a_reg[11] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 12 ) * KC + kr ), a_reg[12] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 13 ) * KC + kr ), a_reg[13] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 14 ) * KC + kr ), a_reg[14] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 15 ) * KC + kr ), a_reg[15] ); + } + for ( ; ( kr + 7 ) < KC; kr += 8 ) + { + a_reg[0] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ) ); + a_reg[1] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ) ); + a_reg[2] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) ) ); + a_reg[3] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) ) ); + a_reg[4] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 4 ) * cs_a ) ) ); + a_reg[5] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 5 ) * cs_a ) ) ); + a_reg[6] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 6 ) * cs_a ) ) ); + a_reg[7] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 7 ) * cs_a ) ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8(mask3, mask4) + MASKED_STORE_PS(0xFF); + } + for( ; ( kr + 3 ) < KC; kr += 4) + { + a_reg[0] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ) ); + a_reg[1] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ) ); + a_reg[2] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) ) ); + a_reg[3] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + MASKED_STORE_PS(0x0F); + } + for( ; ( kr + 1 ) < KC; kr += 2) + { + a_reg[0] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ) ); + a_reg[1] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8(mask3, mask4) + MASKED_STORE_PS(0x03); + } + for( ; ( kr ) < KC; kr += 1) + { + a_reg[0] = _mm512_loadu_ps( (__m512 const *)(a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8(mask3, mask4) + MASKED_STORE_PS(0x01); + } + } + for( ; (ic + 8 - 1) < MC; ic += 8) + { + for( kr = 0; ( kr + 15 ) < KC; kr += 16) + { + a_reg[0] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 0 ) * cs_a ) ) ); + a_reg[1] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 1 ) * cs_a ) ) ); + a_reg[2] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 2 ) * cs_a ) ) ); + a_reg[3] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 3 ) * cs_a ) ) ); + a_reg[4] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 4 ) * cs_a ) ) ); + a_reg[5] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 5 ) * cs_a ) ) ); + a_reg[6] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 6 ) * cs_a ) ) ); + a_reg[7] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 7 ) * cs_a ) ) ); + a_reg[8] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 8 ) * cs_a ) ) ); + a_reg[9] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 9 ) * cs_a ) ) ); + a_reg[10] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 10 ) * cs_a ) ) ); + a_reg[11] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 11 ) * cs_a ) ) ); + a_reg[12] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 12 ) * cs_a ) ) ); + a_reg[13] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 13 ) * cs_a ) ) ); + a_reg[14] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 14 ) * cs_a ) ) ); + a_reg[15] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 15 ) * cs_a ) ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), a_reg[0] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), a_reg[1] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), a_reg[2] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), a_reg[3] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 4 ) * KC + kr ), a_reg[4] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 5 ) * KC + kr ), a_reg[5] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 6 ) * KC + kr ), a_reg[6] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 7 ) * KC + kr ), a_reg[7] ); + } + for( ; ( kr + 7 ) < KC; kr += 8) + { + a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ); + a_reg[1] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ); + a_reg[2] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) ); + a_reg[3] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) ); + a_reg[4] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 4 ) * cs_a ) ); + a_reg[5] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 5 ) * cs_a ) ); + a_reg[6] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 6 ) * cs_a ) ); + a_reg[7] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 7 ) * cs_a ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4(mask1, mask2) + PERMUTE8x8(mask3, mask4) + + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0xFF, a_reg[0] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0xFF, a_reg[1] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0xFF, a_reg[2] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0xFF, a_reg[3] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 4 ) * KC + kr ), 0xFF, a_reg[4] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 5 ) * KC + kr ), 0xFF, a_reg[5] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 6 ) * KC + kr ), 0xFF, a_reg[6] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 7 ) * KC + kr ), 0xFF, a_reg[7] ); + } + for( ; ( kr + 3 ) < KC; kr += 4) + { + a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ); + a_reg[1] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ); + a_reg[2] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) ); + a_reg[3] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x0F, a_reg[0] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x0F, a_reg[1] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x0F, a_reg[2] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0x0F, a_reg[3] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 4 ) * KC + kr ), 0x0F, a_reg[4] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 5 ) * KC + kr ), 0x0F, a_reg[5] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 6 ) * KC + kr ), 0x0F, a_reg[6] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 7 ) * KC + kr ), 0x0F, a_reg[7] ); + } + for( ; ( kr + 1 ) < KC; kr += 2) + { + a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ); + a_reg[1] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x03, a_reg[0] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x03, a_reg[1] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x03, a_reg[2] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0x03, a_reg[3] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 4 ) * KC + kr ), 0x03, a_reg[4] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 5 ) * KC + kr ), 0x03, a_reg[5] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 6 ) * KC + kr ), 0x03, a_reg[6] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 7 ) * KC + kr ), 0x03, a_reg[7] ); + + } + for( ; ( kr ) < KC; kr += 1) + { + a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x01, a_reg[0] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x01, a_reg[1] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x01, a_reg[2] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0x01, a_reg[3] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 4 ) * KC + kr ), 0x01, a_reg[4] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 5 ) * KC + kr ), 0x01, a_reg[5] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 6 ) * KC + kr ), 0x01, a_reg[6] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 7 ) * KC + kr ), 0x01, a_reg[7] ); + } + } + for( ; ( ic + 4 - 1 ) < MC; ic += 4) + { + for( kr = 0; ( kr + 15 ) < KC; kr += 16) + { + a_reg[0] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 0 ) * cs_a ) ) ); + a_reg[1] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 1 ) * cs_a ) ) ); + a_reg[2] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 2 ) * cs_a ) ) ); + a_reg[3] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 3 ) * cs_a ) ) ); + a_reg[4] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 4 ) * cs_a ) ) ); + a_reg[5] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 5 ) * cs_a ) ) ); + a_reg[6] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 6 ) * cs_a ) ) ); + a_reg[7] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 7 ) * cs_a ) ) ); + a_reg[8] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 8 ) * cs_a ) ) ); + a_reg[9] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 9 ) * cs_a ) ) ); + a_reg[10] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 10 ) * cs_a ) ) ); + a_reg[11] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 11 ) * cs_a ) ) ); + a_reg[12] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 12 ) * cs_a ) ) ); + a_reg[13] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 13 ) * cs_a ) ) ); + a_reg[14] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 14 ) * cs_a ) ) ); + a_reg[15] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 15 ) * cs_a ) ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), a_reg[0] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), a_reg[1] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), a_reg[2] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), a_reg[3] ); + } + for( ; ( kr + 7 ) < KC; kr += 8) + { + a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ); + a_reg[1] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ); + a_reg[2] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) ); + a_reg[3] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) ); + a_reg[4] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 4 ) * cs_a ) ); + a_reg[5] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 5 ) * cs_a ) ); + a_reg[6] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 6 ) * cs_a ) ); + a_reg[7] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 7 ) * cs_a ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4(mask1, mask2) + PERMUTE8x8(mask3, mask4) + + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0xFF, a_reg[0] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0xFF, a_reg[1] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0xFF, a_reg[2] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0xFF, a_reg[3] ); + } + for( ; ( kr + 3 ) < KC; kr += 4) + { + a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ); + a_reg[1] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ); + a_reg[2] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) ); + a_reg[3] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x0F, a_reg[0] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x0F, a_reg[1] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x0F, a_reg[2] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0x0F, a_reg[3] ); + } + for( ; ( kr + 1 ) < KC; kr += 2) + { + a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ); + a_reg[1] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x03, a_reg[0] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x03, a_reg[1] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x03, a_reg[2] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0x03, a_reg[3] ); + } + for( ; ( kr ) < KC; kr += 1) + { + a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x01, a_reg[0] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x01, a_reg[1] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x01, a_reg[2] ); + _mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0x01, a_reg[3] ); + } + } + if( m_left ) { + __mmask16 mask = 0xFFFF >> ( 16 - m_left ); + for( kr = 0; ( kr + 15 ) < KC; kr += 16) + { + a_reg[0] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 0 ) * cs_a ) ) ); + a_reg[1] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 1 ) * cs_a ) ) ); + a_reg[2] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 2 ) * cs_a ) ) ); + a_reg[3] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 3 ) * cs_a ) ) ); + a_reg[4] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 4 ) * cs_a ) ) ); + a_reg[5] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 5 ) * cs_a ) ) ); + a_reg[6] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 6 ) * cs_a ) ) ); + a_reg[7] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 7 ) * cs_a ) ) ); + a_reg[8] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 8 ) * cs_a ) ) ); + a_reg[9] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 9 ) * cs_a ) ) ); + a_reg[10] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 10 ) * cs_a ) ) ); + a_reg[11] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 11 ) * cs_a ) ) ); + a_reg[12] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 12 ) * cs_a ) ) ); + a_reg[13] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 13 ) * cs_a ) ) ); + a_reg[14] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 14 ) * cs_a ) ) ); + a_reg[15] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 15 ) * cs_a ) ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + + switch( m_left ) + { + case 3: + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), a_reg[0] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), a_reg[1] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), a_reg[2] ); + break; + + case 2: + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), a_reg[0] ); + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), a_reg[1] ); + break; + + case 1: + _mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), a_reg[0] ); + break; + } + } + for( ; ( kr + 7 ) < KC; kr += 8) + { + a_reg[0] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 0 ) * cs_a ) ) ); + a_reg[1] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 1 ) * cs_a ) ) ); + a_reg[2] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 2 ) * cs_a ) ) ); + a_reg[3] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 3 ) * cs_a ) ) ); + a_reg[4] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 4 ) * cs_a ) ) ); + a_reg[5] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 5 ) * cs_a ) ) ); + a_reg[6] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 6 ) * cs_a ) ) ); + a_reg[7] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 7 ) * cs_a ) ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + + switch( m_left ) + { + case 3: + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0xFF, a_reg[0]); + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0xFF, a_reg[1]); + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0xFF, a_reg[2]); + break; + + case 2: + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0xFF, a_reg[0]); + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0xFF, a_reg[1]); + break; + + case 1: + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0xFF, a_reg[0]); + break; + } + } + for( ; ( kr + 3 ) < KC; kr += 4) + { + a_reg[0] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 0 ) * cs_a ) ) ); + a_reg[1] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 1 ) * cs_a ) ) ); + a_reg[2] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 2 ) * cs_a ) ) ); + a_reg[3] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 3 ) * cs_a ) ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + + switch( m_left ) + { + case 3: + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x0F, a_reg[0]); + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x0F, a_reg[1]); + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x0F, a_reg[2]); + break; + + case 2: + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x0F, a_reg[0]); + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x0F, a_reg[1]); + break; + + case 1: + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x0F, a_reg[0]); + break; + } + } + for( ; ( kr + 1 ) < KC; kr += 2) + { + a_reg[0] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 0 ) * cs_a ) ) ); + a_reg[1] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 1 ) * cs_a ) ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + + switch( m_left ) + { + case 3: + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x03, a_reg[0]); + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x03, a_reg[1]); + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x03, a_reg[2]); + break; + + case 2: + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x03, a_reg[0]); + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x03, a_reg[1]); + break; + + case 1: + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x03, a_reg[0]); + break; + } + } + for( ; ( kr ) < KC; kr += 1) + { + a_reg[0] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) + + ( ( kr + 0 ) * cs_a ) ) ); + + UNPACKLO_PS16 + UNPACKHI_PS16 + SHUFFLE_64x2 + PERMUTE4x4( mask1, mask2 ) + PERMUTE8x8( mask3, mask4 ) + + switch( m_left ) + { + case 3: + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x01, a_reg[0]); + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x01, a_reg[1]); + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x01, a_reg[2]); + break; + + case 2: + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x01, a_reg[0]); + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x01, a_reg[1]); + break; + + case 1: + _mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x01, a_reg[0]); + break; + } + } + } + *rs_p = KC; + *cs_p = 1; +} +#endif