amd/blis
1
0
Fork 0
mirror of https://github.com/amd/blis.git synced 2026-07-14 11:07:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Transpose Kernel for A matrix in F32F32f32Of32

Browse Source 
- Implemented the AVX512 packA kernel for col major inputs in F32 API
 - Removed the work arounds for n = 1,  mtag_a = PACK case, where the execution was
   being directed to GEMM instead of GEMV.

Change-Id: I6fb700d96069213a762e8a83a209c5388a91050f
This commit is contained in:
varshav2
2024-09-17 22:14:27 +05:30
committed by Edward Smyth
parent df375e5557
commit e06fa8b0d2
6 changed files with 857 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
addon/aocl_gemm/aocl_gemm.h
View File

@@ -53,6 +53,7 @@
#include "lpgemm_packa_s8.h"
#include "lpgemm_packb_s8.h"
#include "lpgemm_packb_s8s16.h"
#include "lpgemm_pack_f32.h"
#include "lpgemm_jit_typedefs.h"
#ifdef LPGEMM_BF16_JIT
#include "lpgemm_jit_c_connector.h"

1
addon/aocl_gemm/aocl_gemm_f32f32f32of32.c
View File

@@ -168,7 +168,6 @@ AOCL_GEMM_MATMUL(float,float,float,float,f32f32f32of32)
bli_pba_rntm_set_pba( &rntm_g );
lpgemm_cntx_t* lcntx_g = lpgemm_get_global_cntx_obj( F32F32F32OF32 );
#ifdef BLIS_ENABLE_OPENMP
// The lpgemm_cntx_t argument will be NULL for f32 since it still uses
// BLIS cntx_t internally. Its a workaround for now and will be replaced

1
addon/aocl_gemm/config/lpgemm_config.c
View File

@@ -45,6 +45,7 @@
#include "lpgemm_packa_s8.h"
#include "lpgemm_packb_s8.h"
#include "lpgemm_packb_s8s16.h"
#include "lpgemm_pack_f32.h"
static lpgemm_cntx_t global_cntx_t_list[AOCL_OPERATION_TYPE_LEN] \
__attribute__((aligned(64))); //Only one op type supported now.

83
addon/aocl_gemm/frame/f32f32f32/lpgemm_f32f32f32.c
View File

@@ -38,6 +38,7 @@
#include "lpgemm_utils.h"
#include "lpgemm_thrinfo_utils.h"
#include "lpgemm_kernels.h"
#include "lpgemm_pack_f32.h"
// Kernel function prototypes
typedef void (*lpgemm_rowvar_f32)
@@ -96,7 +97,6 @@ LPGEMV(float, float, float, f32f32f32of32)
const float* a_use = (float*)a;
inc_t rs_a_use = rs_a;
inc_t cs_a_use = cs_a;
inc_t ps_a_use;
float* b_use = (float*)b;
inc_t rs_b_use = rs_b;
@@ -129,15 +129,13 @@ LPGEMV(float, float, float, f32f32f32of32)
thrinfo_t thread_jc;
thrinfo_t thread_ic;
lpgemm_gen_thrinfo(thread, &thread_jc, &thread_ic);
if(n == 1)
{
float* pack_b_buffer_f32f32f32of32;
//TODO: AVX2 support need to be added
// Increased MR from 6 to 16 to make use of 32 ZMM registers
dim_t MR = 16;
// Pack B matrix if rs_b > 1
if( ( mtag_b == PACK ) && ( rs_b != 1 ) )
{
@@ -175,7 +173,7 @@ LPGEMV(float, float, float, f32f32f32of32)
c_use = c + ic * rs_c;
post_ops_attr.post_op_c_i = ic;
if( mtag_a == PACK )
if( mtag_a == PACK && cs_a != 1 )
{
mem_a_size_req = sizeof(float) * mc0 * k;
lpgemm_alloc_mem_panel
@@ -185,19 +183,13 @@ LPGEMV(float, float, float, f32f32f32of32)
);
pack_a_buffer_f32f32f32of32 = ( float* )bli_mem_buffer( &mem_a );
rs_a_use = 1;
cs_a_use = MR;
ps_a_use = MR * k;
lpgemm_pack_a_f32f32f32of32
(
a_use,
packa_mr16_f32f32f32of32_col_major
(
pack_a_buffer_f32f32f32of32,
a_use, rs_a, cs_a,
mc0, k,
rs_a, cs_a, ps_a_use, MR,
cntx
);
&rs_a_use, &cs_a_use
);
a_use = pack_a_buffer_f32f32f32of32;
}
@@ -224,14 +216,38 @@ LPGEMV(float, float, float, f32f32f32of32)
}
}
else
{
{
// Compute the JC loop thread range for the current thread.
dim_t jc_start, jc_end;
thread_jc.n_way = ( thread_jc.n_way == 1 ) ?
( thread->n_threads ) : ( thread_jc.n_way );
thread_jc.work_id = thread->tid;
bli_thread_range_sub(&thread_jc, n, NR, FALSE, &jc_start, &jc_end);
if ( mtag_a == PACK )
{
mem_a_size_req = sizeof( float ) * k;
lpgemm_alloc_mem_panel
(
mem_a_size_req, BLIS_BUFFER_FOR_GEN_USE,
&mem_a, rntm
);
pack_a_buffer_f32f32f32of32 =
( float* ) bli_mem_buffer( &mem_a );
packa_mr16_f32f32f32of32_col_major
(
pack_a_buffer_f32f32f32of32,
a_use, rs_a, cs_a,
1, k,
&rs_a_use, &cs_a_use
);
a_use = pack_a_buffer_f32f32f32of32;
}
for (dim_t jc = jc_start; jc < jc_end; jc += NC)
{
dim_t nc0 = bli_min((jc_end - jc), NC);
@@ -253,7 +269,7 @@ LPGEMV(float, float, float, f32f32f32of32)
rs_b_use = NR;
cs_b_use = 1;
}
else if (mtag_b == PACK)
else if (mtag_b == PACK)
{
// nc0 needs to be a multiple of 16 since this gives maximum
// vectorization. Packing B always results in buffers with width
@@ -263,20 +279,20 @@ LPGEMV(float, float, float, f32f32f32of32)
mem_b_size_req = sizeof( float ) * nc0_updated * k;
n_sub_updated = nc0_updated;
lpgemm_alloc_mem_panel
(
mem_b_size_req, BLIS_BUFFER_FOR_B_PANEL,
&mem_b, rntm
);
pack_b_buffer_f32f32f32of32 =
( float* ) bli_mem_buffer( &mem_b );
for ( dim_t pc = 0; pc < k; pc += KC )
{
dim_t kc0 = bli_min( ( k - pc ), KC );
// Set the strides for pack buffer.
rs_b_use = NR;
cs_b_use = 1;
@@ -289,7 +305,7 @@ LPGEMV(float, float, float, f32f32f32of32)
nc0 , kc0,
rs_b, cs_b, ( NR * ps_b_use ), NR,
cntx
);
);
}
b_use = pack_b_buffer_f32f32f32of32;
}
@@ -297,7 +313,7 @@ LPGEMV(float, float, float, f32f32f32of32)
{
b_use = (float*) b + jc * cs_b;
}
//update post-op pointer
post_ops_attr.post_op_c_j = jc;
@@ -305,7 +321,7 @@ LPGEMV(float, float, float, f32f32f32of32)
lpgemv_m_one_f32f32f32of32
(
nc0, k,
a, rs_a, cs_a, mtag_a,
a_use, rs_a_use, cs_a_use, mtag_a,
b_use, rs_b_use, cs_b_use, mtag_b,
c_use, rs_c, cs_c,
alpha, beta,
@@ -336,10 +352,7 @@ LPGEMM_5LOOP(float, float, float, f32f32f32of32)
#ifdef BLIS_KERNELS_ZEN4
// Handle using LPGEMV when m or/and n equal to 1
// The avx512 check will be removed when avx2 kernels added in future
//ToDo: with trasnsA row storage and transB column storage, the packed matrices will be in col stored row access
//which will give error in the computation. Hence, for now redirecting those cases to GEMM instead of GEMV to avoid the errors.
if ( ( ( m == 1 ) || ( n == 1 ) ) && (bli_cpuid_is_avx512_supported() == TRUE) &&
( mtag_a != PACK ) )
if ( ( ( m == 1 ) || ( n == 1 ) ) && (bli_cpuid_is_avx512_supported() == TRUE) )
{
lpgemv_rowvar_f32f32f32of32(m, n, k,
a, rs_a, cs_a, mtag_a,
@@ -355,16 +368,6 @@ LPGEMM_5LOOP(float, float, float, f32f32f32of32)
return;
}
#endif
//ToDo: In case of transA with row storage, the padding will not be done if mtag_a is enabled by user.
//This would give a seg fault. Hence, adding the condition here so that this will be taken care.
if( ( n == 1 ) && ( mtag_a == PACK ) ) {
if(mtag_b == REORDERED) {
rs_b = 1;
cs_b = 1;
}
mtag_b = PACK;
}
// Query the global cntx.
cntx_t* cntx = bli_gks_query_cntx();

50
addon/aocl_gemm/kernels/f32f32f32/lpgemm_pack_f32.h Normal file
View File

@@ -0,0 +1,50 @@
/*
BLIS
An object-based framework for developing high-performance BLAS-like
libraries.
Copyright (C) 2024, Advanced Micro Devices, Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
- Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
- Neither the name(s) of the copyright holder(s) nor the names of its
contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef BLIS_GEMM_F32_PACKA
#define BLIS_GEMM_F32_PACKA
void packa_mr16_f32f32f32of32_col_major
(
float* pack_a_buffer,
const float* a,
const dim_t rs_a,
const dim_t cs_a,
const dim_t MC,
const dim_t KC,
dim_t* rs_p,
dim_t* cs_p
);
#endif

762
kernels/zen4/lpgemm/f32f32f32/lpgemm_pack_a_f32_amd512vnni.c Normal file
View File

@@ -0,0 +1,762 @@
/*
BLIS
An object-based framework for developing high-performance BLAS-like
libraries.
Copyright (C) 2024, Advanced Micro Devices, Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
- Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
- Redistributions in binarsy form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
- Neither the name(s) of the copyright holder(s) nor the names of its
contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <immintrin.h>
#include <string.h>
#include "blis.h"
#ifdef BLIS_ADDON_LPGEMM
#define UNPACKLO_PS16 \
b_reg[0] = _mm512_unpacklo_ps(a_reg[0], a_reg[1]); \
b_reg[1] = _mm512_unpacklo_ps(a_reg[2], a_reg[3]); \
b_reg[2] = _mm512_unpacklo_ps(a_reg[4], a_reg[5]); \
b_reg[3] = _mm512_unpacklo_ps(a_reg[6], a_reg[7]); \
b_reg[4] = _mm512_unpacklo_ps(a_reg[8], a_reg[9]); \
b_reg[5] = _mm512_unpacklo_ps(a_reg[10], a_reg[11]); \
b_reg[6] = _mm512_unpacklo_ps(a_reg[12], a_reg[13]); \
b_reg[7] = _mm512_unpacklo_ps(a_reg[14], a_reg[15]);
#define UNPACKHI_PS16 \
b_reg[8] = _mm512_unpackhi_ps(a_reg[0], a_reg[1]); \
b_reg[9] = _mm512_unpackhi_ps(a_reg[2], a_reg[3]); \
b_reg[10] = _mm512_unpackhi_ps(a_reg[4], a_reg[5]); \
b_reg[11] = _mm512_unpackhi_ps(a_reg[6], a_reg[7]); \
b_reg[12] = _mm512_unpackhi_ps(a_reg[8], a_reg[9]); \
b_reg[13] = _mm512_unpackhi_ps(a_reg[10], a_reg[11]); \
b_reg[14] = _mm512_unpackhi_ps(a_reg[12], a_reg[13]); \
b_reg[15] = _mm512_unpackhi_ps(a_reg[14], a_reg[15]);
#define SHUFFLE_64x2 \
a_reg[0] = _mm512_shuffle_ps(b_reg[0], b_reg[1], 0x44); \
a_reg[1] = _mm512_shuffle_ps(b_reg[0], b_reg[1], 0xEE); \
a_reg[2] = _mm512_shuffle_ps(b_reg[2], b_reg[3], 0x44); \
a_reg[3] = _mm512_shuffle_ps(b_reg[2], b_reg[3], 0xEE); \
\
a_reg[4] = _mm512_shuffle_ps(b_reg[4], b_reg[5], 0x44); \
a_reg[5] = _mm512_shuffle_ps(b_reg[4], b_reg[5], 0xEE); \
a_reg[6] = _mm512_shuffle_ps(b_reg[6], b_reg[7], 0x44); \
a_reg[7] = _mm512_shuffle_ps(b_reg[6], b_reg[7], 0xEE); \
\
a_reg[8] = _mm512_shuffle_ps(b_reg[8], b_reg[9], 0x44); \
a_reg[9] = _mm512_shuffle_ps(b_reg[8], b_reg[9], 0xEE); \
a_reg[10] = _mm512_shuffle_ps(b_reg[10], b_reg[11], 0x44); \
a_reg[11] = _mm512_shuffle_ps(b_reg[10], b_reg[11], 0xEE); \
\
a_reg[12] = _mm512_shuffle_ps(b_reg[12], b_reg[13], 0x44); \
a_reg[13] = _mm512_shuffle_ps(b_reg[12], b_reg[13], 0xEE); \
a_reg[14] = _mm512_shuffle_ps(b_reg[14], b_reg[15], 0x44); \
a_reg[15] = _mm512_shuffle_ps(b_reg[14], b_reg[15], 0xEE);
#define MASKED_STORE_PS(mask) \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+0) * KC + kr ), mask, a_reg[0]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+1) * KC + kr ), mask, a_reg[1]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+2) * KC + kr ), mask, a_reg[2]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+3) * KC + kr ), mask, a_reg[3]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+4) * KC + kr ), mask, a_reg[4]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+5) * KC + kr ), mask, a_reg[5]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+6) * KC + kr ), mask, a_reg[6]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+7) * KC + kr ), mask, a_reg[7]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+8) * KC + kr ), mask, a_reg[8]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+9) * KC + kr ), mask, a_reg[9]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+10) * KC + kr ), mask, a_reg[10]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+11) * KC + kr ), mask, a_reg[11]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+12) * KC + kr ), mask, a_reg[12]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+13) * KC + kr ), mask, a_reg[13]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+14) * KC + kr ), mask, a_reg[14]); \
_mm512_mask_storeu_ps((pack_a_buffer + (ic+15) * KC + kr ), mask, a_reg[15]);
#define PERMUTE4x4( mask1, mask2 ) \
b_reg[0] = _mm512_permutex2var_ps( a_reg[0], mask1, a_reg[2] ); \
b_reg[1] = _mm512_permutex2var_ps( a_reg[1], mask1, a_reg[3] ); \
b_reg[2] = _mm512_permutex2var_ps( a_reg[8], mask1, a_reg[10] ); \
b_reg[3] = _mm512_permutex2var_ps( a_reg[9], mask1, a_reg[11] ); \
\
b_reg[4] = _mm512_permutex2var_ps( a_reg[4], mask1, a_reg[6]); \
b_reg[5] = _mm512_permutex2var_ps( a_reg[5], mask1, a_reg[7]); \
b_reg[6] = _mm512_permutex2var_ps( a_reg[12], mask1, a_reg[14]); \
b_reg[7] = _mm512_permutex2var_ps( a_reg[13], mask1, a_reg[15]); \
\
b_reg[8] = _mm512_permutex2var_ps( a_reg[0], mask2, a_reg[2]); \
b_reg[9] = _mm512_permutex2var_ps( a_reg[1], mask2, a_reg[3]); \
b_reg[10] = _mm512_permutex2var_ps( a_reg[8], mask2, a_reg[10]); \
b_reg[11] = _mm512_permutex2var_ps( a_reg[9], mask2, a_reg[11]); \
\
b_reg[12] = _mm512_permutex2var_ps( a_reg[4], mask2, a_reg[6]); \
b_reg[13] = _mm512_permutex2var_ps( a_reg[5], mask2, a_reg[7]); \
b_reg[14] = _mm512_permutex2var_ps( a_reg[12], mask2, a_reg[14]); \
b_reg[15] = _mm512_permutex2var_ps( a_reg[13], mask2, a_reg[15]);
#define PERMUTE8x8( mask3, mask4 ) \
a_reg[0] = _mm512_permutex2var_ps( b_reg[0], mask3, b_reg[4]); \
a_reg[1] = _mm512_permutex2var_ps( b_reg[1], mask3, b_reg[5]); \
a_reg[2] = _mm512_permutex2var_ps( b_reg[2], mask3, b_reg[6]); \
a_reg[3] = _mm512_permutex2var_ps( b_reg[3], mask3, b_reg[7]); \
\
a_reg[4] = _mm512_permutex2var_ps( b_reg[0], mask4, b_reg[4]); \
a_reg[5] = _mm512_permutex2var_ps( b_reg[1], mask4, b_reg[5]); \
a_reg[6] = _mm512_permutex2var_ps( b_reg[2], mask4, b_reg[6]); \
a_reg[7] = _mm512_permutex2var_ps( b_reg[3], mask4, b_reg[7]); \
\
a_reg[8] = _mm512_permutex2var_ps( b_reg[8], mask3, b_reg[12]); \
a_reg[9] = _mm512_permutex2var_ps( b_reg[9], mask3, b_reg[13]); \
a_reg[10] = _mm512_permutex2var_ps( b_reg[10], mask3, b_reg[14]); \
a_reg[11] = _mm512_permutex2var_ps( b_reg[11], mask3, b_reg[15]); \
\
a_reg[12] = _mm512_permutex2var_ps( b_reg[8], mask4, b_reg[12]); \
a_reg[13] = _mm512_permutex2var_ps( b_reg[9], mask4, b_reg[13]); \
a_reg[14] = _mm512_permutex2var_ps( b_reg[10], mask4, b_reg[14]); \
a_reg[15] = _mm512_permutex2var_ps( b_reg[11], mask4, b_reg[15]);
void packa_mr16_f32f32f32of32_col_major
(
float* pack_a_buffer,
const float* a,
const dim_t rs_a,
const dim_t cs_a,
const dim_t MC,
const dim_t KC,
dim_t* rs_p,
dim_t* cs_p
)
{
dim_t MR = 16;
dim_t ic, kr;
dim_t m_left = MC % 4;
__m512 a_reg[16], b_reg[16];
__m512i mask1 = _mm512_set_epi32( 0x17, 0x16, 0x15, 0x14,
0x07, 0x06, 0x05, 0x04,
0x13, 0x12, 0x11, 0x10,
0x03, 0x02, 0x01, 0x00 );
__m512i mask2 = _mm512_set_epi32( 0x1F, 0x1E, 0x1D, 0x1C,
0x0F, 0x0E, 0x0D, 0x0C,
0x1B, 0x1A, 0x19, 0x18,
0x0B, 0x0A, 0x9, 0x08 );
__m512i mask3 = _mm512_set_epi32( 0x17, 0x16, 0x15, 0x14, 0x13, 0x12, 0x11, 0x10,
0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00 );
__m512i mask4 = _mm512_set_epi32( 0x1F, 0x1E, 0x1D, 0x1C, 0x1B, 0x1A, 0x19, 0x18,
0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 );
// These registers are set with zeroes to avoid compiler warnings
// To-DO: TO be removed when pack code is optimized for fringe cases.
a_reg[0] = _mm512_setzero_ps();
a_reg[1] = _mm512_setzero_ps();
a_reg[2] = _mm512_setzero_ps();
a_reg[3] = _mm512_setzero_ps();
a_reg[4] = _mm512_setzero_ps();
a_reg[5] = _mm512_setzero_ps();
a_reg[6] = _mm512_setzero_ps();
a_reg[7] = _mm512_setzero_ps();
a_reg[8] = _mm512_setzero_ps();
a_reg[9] = _mm512_setzero_ps();
a_reg[10] = _mm512_setzero_ps();
a_reg[11] = _mm512_setzero_ps();
a_reg[12] = _mm512_setzero_ps();
a_reg[13] = _mm512_setzero_ps();
a_reg[14] = _mm512_setzero_ps();
a_reg[15] = _mm512_setzero_ps();
for( ic = 0; ( ic + MR - 1 ) < MC; ic += MR)
{
for( kr = 0; ( kr + 15 ) < KC; kr += 16)
{
a_reg[0] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ) );
a_reg[1] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ) );
a_reg[2] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) ) );
a_reg[3] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) ) );
a_reg[4] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 4 ) * cs_a ) ) );
a_reg[5] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 5 ) * cs_a ) ) );
a_reg[6] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 6 ) * cs_a ) ) );
a_reg[7] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 7 ) * cs_a ) ) );
a_reg[8] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 8 ) * cs_a ) ) );
a_reg[9] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) + ( ( kr + 9 ) * cs_a ) ) );
a_reg[10] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 10 ) * cs_a ) ) );
a_reg[11] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 11 ) * cs_a ) ) );
a_reg[12] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 12 ) * cs_a ) ) );
a_reg[13] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 13 ) * cs_a ) ) );
a_reg[14] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 14 ) * cs_a ) ) );
a_reg[15] = _mm512_loadu_ps( (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 15 ) * cs_a ) ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 );
PERMUTE8x8( mask3, mask4 )
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), a_reg[0] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), a_reg[1] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), a_reg[2] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), a_reg[3] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 4 ) * KC + kr ), a_reg[4] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 5 ) * KC + kr ), a_reg[5] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 6 ) * KC + kr ), a_reg[6] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 7 ) * KC + kr ), a_reg[7] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 8 ) * KC + kr ), a_reg[8] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 9 ) * KC + kr ), a_reg[9] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 10 ) * KC + kr ), a_reg[10] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 11 ) * KC + kr ), a_reg[11] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 12 ) * KC + kr ), a_reg[12] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 13 ) * KC + kr ), a_reg[13] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 14 ) * KC + kr ), a_reg[14] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 15 ) * KC + kr ), a_reg[15] );
}
for ( ; ( kr + 7 ) < KC; kr += 8 )
{
a_reg[0] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ) );
a_reg[1] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ) );
a_reg[2] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) ) );
a_reg[3] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) ) );
a_reg[4] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 4 ) * cs_a ) ) );
a_reg[5] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 5 ) * cs_a ) ) );
a_reg[6] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 6 ) * cs_a ) ) );
a_reg[7] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 7 ) * cs_a ) ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8(mask3, mask4)
MASKED_STORE_PS(0xFF);
}
for( ; ( kr + 3 ) < KC; kr += 4)
{
a_reg[0] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ) );
a_reg[1] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ) );
a_reg[2] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) ) );
a_reg[3] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
MASKED_STORE_PS(0x0F);
}
for( ; ( kr + 1 ) < KC; kr += 2)
{
a_reg[0] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ) );
a_reg[1] = _mm512_loadu_ps( (__m512 const *)( a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8(mask3, mask4)
MASKED_STORE_PS(0x03);
}
for( ; ( kr ) < KC; kr += 1)
{
a_reg[0] = _mm512_loadu_ps( (__m512 const *)(a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8(mask3, mask4)
MASKED_STORE_PS(0x01);
}
}
for( ; (ic + 8 - 1) < MC; ic += 8)
{
for( kr = 0; ( kr + 15 ) < KC; kr += 16)
{
a_reg[0] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 0 ) * cs_a ) ) );
a_reg[1] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 1 ) * cs_a ) ) );
a_reg[2] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 2 ) * cs_a ) ) );
a_reg[3] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 3 ) * cs_a ) ) );
a_reg[4] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 4 ) * cs_a ) ) );
a_reg[5] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 5 ) * cs_a ) ) );
a_reg[6] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 6 ) * cs_a ) ) );
a_reg[7] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 7 ) * cs_a ) ) );
a_reg[8] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 8 ) * cs_a ) ) );
a_reg[9] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 9 ) * cs_a ) ) );
a_reg[10] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 10 ) * cs_a ) ) );
a_reg[11] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 11 ) * cs_a ) ) );
a_reg[12] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 12 ) * cs_a ) ) );
a_reg[13] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 13 ) * cs_a ) ) );
a_reg[14] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 14 ) * cs_a ) ) );
a_reg[15] = _mm512_maskz_loadu_ps( 0xFF, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 15 ) * cs_a ) ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), a_reg[0] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), a_reg[1] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), a_reg[2] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), a_reg[3] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 4 ) * KC + kr ), a_reg[4] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 5 ) * KC + kr ), a_reg[5] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 6 ) * KC + kr ), a_reg[6] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 7 ) * KC + kr ), a_reg[7] );
}
for( ; ( kr + 7 ) < KC; kr += 8)
{
a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) );
a_reg[1] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) );
a_reg[2] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) );
a_reg[3] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) );
a_reg[4] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 4 ) * cs_a ) );
a_reg[5] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 5 ) * cs_a ) );
a_reg[6] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 6 ) * cs_a ) );
a_reg[7] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 7 ) * cs_a ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4(mask1, mask2)
PERMUTE8x8(mask3, mask4)
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0xFF, a_reg[0] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0xFF, a_reg[1] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0xFF, a_reg[2] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0xFF, a_reg[3] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 4 ) * KC + kr ), 0xFF, a_reg[4] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 5 ) * KC + kr ), 0xFF, a_reg[5] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 6 ) * KC + kr ), 0xFF, a_reg[6] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 7 ) * KC + kr ), 0xFF, a_reg[7] );
}
for( ; ( kr + 3 ) < KC; kr += 4)
{
a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) );
a_reg[1] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) );
a_reg[2] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) );
a_reg[3] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x0F, a_reg[0] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x0F, a_reg[1] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x0F, a_reg[2] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0x0F, a_reg[3] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 4 ) * KC + kr ), 0x0F, a_reg[4] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 5 ) * KC + kr ), 0x0F, a_reg[5] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 6 ) * KC + kr ), 0x0F, a_reg[6] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 7 ) * KC + kr ), 0x0F, a_reg[7] );
}
for( ; ( kr + 1 ) < KC; kr += 2)
{
a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) );
a_reg[1] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x03, a_reg[0] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x03, a_reg[1] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x03, a_reg[2] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0x03, a_reg[3] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 4 ) * KC + kr ), 0x03, a_reg[4] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 5 ) * KC + kr ), 0x03, a_reg[5] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 6 ) * KC + kr ), 0x03, a_reg[6] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 7 ) * KC + kr ), 0x03, a_reg[7] );
}
for( ; ( kr ) < KC; kr += 1)
{
a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x01, a_reg[0] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x01, a_reg[1] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x01, a_reg[2] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0x01, a_reg[3] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 4 ) * KC + kr ), 0x01, a_reg[4] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 5 ) * KC + kr ), 0x01, a_reg[5] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 6 ) * KC + kr ), 0x01, a_reg[6] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 7 ) * KC + kr ), 0x01, a_reg[7] );
}
}
for( ; ( ic + 4 - 1 ) < MC; ic += 4)
{
for( kr = 0; ( kr + 15 ) < KC; kr += 16)
{
a_reg[0] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 0 ) * cs_a ) ) );
a_reg[1] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 1 ) * cs_a ) ) );
a_reg[2] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 2 ) * cs_a ) ) );
a_reg[3] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 3 ) * cs_a ) ) );
a_reg[4] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 4 ) * cs_a ) ) );
a_reg[5] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 5 ) * cs_a ) ) );
a_reg[6] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 6 ) * cs_a ) ) );
a_reg[7] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 7 ) * cs_a ) ) );
a_reg[8] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 8 ) * cs_a ) ) );
a_reg[9] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 9 ) * cs_a ) ) );
a_reg[10] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 10 ) * cs_a ) ) );
a_reg[11] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 11 ) * cs_a ) ) );
a_reg[12] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 12 ) * cs_a ) ) );
a_reg[13] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 13 ) * cs_a ) ) );
a_reg[14] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 14 ) * cs_a ) ) );
a_reg[15] = _mm512_maskz_loadu_ps ( 0x0F, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 15 ) * cs_a ) ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), a_reg[0] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), a_reg[1] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), a_reg[2] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), a_reg[3] );
}
for( ; ( kr + 7 ) < KC; kr += 8)
{
a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) );
a_reg[1] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) );
a_reg[2] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) );
a_reg[3] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) );
a_reg[4] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 4 ) * cs_a ) );
a_reg[5] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 5 ) * cs_a ) );
a_reg[6] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 6 ) * cs_a ) );
a_reg[7] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 7 ) * cs_a ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4(mask1, mask2)
PERMUTE8x8(mask3, mask4)
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0xFF, a_reg[0] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0xFF, a_reg[1] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0xFF, a_reg[2] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0xFF, a_reg[3] );
}
for( ; ( kr + 3 ) < KC; kr += 4)
{
a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) );
a_reg[1] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) );
a_reg[2] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 2 ) * cs_a ) );
a_reg[3] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 3 ) * cs_a ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x0F, a_reg[0] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x0F, a_reg[1] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x0F, a_reg[2] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0x0F, a_reg[3] );
}
for( ; ( kr + 1 ) < KC; kr += 2)
{
a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) );
a_reg[1] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 1 ) * cs_a ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x03, a_reg[0] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x03, a_reg[1] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x03, a_reg[2] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0x03, a_reg[3] );
}
for( ; ( kr ) < KC; kr += 1)
{
a_reg[0] = (__m512)_mm512_maskz_loadu_ps( 0xFF, a + ( ic * rs_a ) + ( ( kr + 0 ) * cs_a ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x01, a_reg[0] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x01, a_reg[1] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x01, a_reg[2] );
_mm512_mask_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 3 ) * KC + kr ), 0x01, a_reg[3] );
}
}
if( m_left ) {
__mmask16 mask = 0xFFFF >> ( 16 - m_left );
for( kr = 0; ( kr + 15 ) < KC; kr += 16)
{
a_reg[0] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 0 ) * cs_a ) ) );
a_reg[1] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 1 ) * cs_a ) ) );
a_reg[2] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 2 ) * cs_a ) ) );
a_reg[3] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 3 ) * cs_a ) ) );
a_reg[4] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 4 ) * cs_a ) ) );
a_reg[5] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 5 ) * cs_a ) ) );
a_reg[6] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 6 ) * cs_a ) ) );
a_reg[7] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 7 ) * cs_a ) ) );
a_reg[8] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 8 ) * cs_a ) ) );
a_reg[9] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 9 ) * cs_a ) ) );
a_reg[10] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 10 ) * cs_a ) ) );
a_reg[11] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 11 ) * cs_a ) ) );
a_reg[12] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 12 ) * cs_a ) ) );
a_reg[13] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 13 ) * cs_a ) ) );
a_reg[14] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 14 ) * cs_a ) ) );
a_reg[15] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 15 ) * cs_a ) ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
switch( m_left )
{
case 3:
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), a_reg[0] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), a_reg[1] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), a_reg[2] );
break;
case 2:
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), a_reg[0] );
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), a_reg[1] );
break;
case 1:
_mm512_storeu_ps( (__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), a_reg[0] );
break;
}
}
for( ; ( kr + 7 ) < KC; kr += 8)
{
a_reg[0] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 0 ) * cs_a ) ) );
a_reg[1] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 1 ) * cs_a ) ) );
a_reg[2] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 2 ) * cs_a ) ) );
a_reg[3] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 3 ) * cs_a ) ) );
a_reg[4] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 4 ) * cs_a ) ) );
a_reg[5] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 5 ) * cs_a ) ) );
a_reg[6] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 6 ) * cs_a ) ) );
a_reg[7] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 7 ) * cs_a ) ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
switch( m_left )
{
case 3:
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0xFF, a_reg[0]);
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0xFF, a_reg[1]);
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0xFF, a_reg[2]);
break;
case 2:
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0xFF, a_reg[0]);
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0xFF, a_reg[1]);
break;
case 1:
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0xFF, a_reg[0]);
break;
}
}
for( ; ( kr + 3 ) < KC; kr += 4)
{
a_reg[0] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 0 ) * cs_a ) ) );
a_reg[1] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 1 ) * cs_a ) ) );
a_reg[2] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 2 ) * cs_a ) ) );
a_reg[3] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 3 ) * cs_a ) ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
switch( m_left )
{
case 3:
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x0F, a_reg[0]);
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x0F, a_reg[1]);
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x0F, a_reg[2]);
break;
case 2:
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x0F, a_reg[0]);
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x0F, a_reg[1]);
break;
case 1:
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x0F, a_reg[0]);
break;
}
}
for( ; ( kr + 1 ) < KC; kr += 2)
{
a_reg[0] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 0 ) * cs_a ) ) );
a_reg[1] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 1 ) * cs_a ) ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
switch( m_left )
{
case 3:
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x03, a_reg[0]);
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x03, a_reg[1]);
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x03, a_reg[2]);
break;
case 2:
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x03, a_reg[0]);
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x03, a_reg[1]);
break;
case 1:
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x03, a_reg[0]);
break;
}
}
for( ; ( kr ) < KC; kr += 1)
{
a_reg[0] = _mm512_maskz_loadu_ps ( mask, (__m512 const *) ( a + ( ic * rs_a ) +
( ( kr + 0 ) * cs_a ) ) );
UNPACKLO_PS16
UNPACKHI_PS16
SHUFFLE_64x2
PERMUTE4x4( mask1, mask2 )
PERMUTE8x8( mask3, mask4 )
switch( m_left )
{
case 3:
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x01, a_reg[0]);
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x01, a_reg[1]);
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 2 ) * KC + kr ), 0x01, a_reg[2]);
break;
case 2:
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x01, a_reg[0]);
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 1 ) * KC + kr ), 0x01, a_reg[1]);
break;
case 1:
_mm512_mask_storeu_ps((__m512 *)( pack_a_buffer + ( ic + 0 ) * KC + kr ), 0x01, a_reg[0]);
break;
}
}
}
*rs_p = KC;
*cs_p = 1;
}
#endif