diff --git a/LICENSE b/LICENSE index 1c6845375..b9cde54b8 100644 --- a/LICENSE +++ b/LICENSE @@ -15,7 +15,7 @@ copyright info. All parties provide their portions of the code under the Copyright (C) 2018, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP -Copyright (C) 2018, Advanced Micro Devices, Inc. +Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/build/bli_config.h.in b/build/bli_config.h.in index e01f9beaa..1bb2ef28b 100644 --- a/build/bli_config.h.in +++ b/build/bli_config.h.in @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/build/config.mk.in b/build/config.mk.in index 34f1931a4..388b95b09 100644 --- a/build/config.mk.in +++ b/build/config.mk.in @@ -92,6 +92,7 @@ CC := @CC@ # Important C compiler ranges. GCC_OT_4_9_0 := @gcc_older_than_4_9_0@ GCC_OT_6_1_0 := @gcc_older_than_6_1_0@ +GCC_OT_9_1_0 := @gcc_older_than_9_1_0@ # The C++ compiler. NOTE: A C++ is typically not needed. CXX := @CXX@ diff --git a/build/detect/config/config_detect.c b/build/detect/config/config_detect.c index 12b93162a..6d59d6625 100644 --- a/build/detect/config/config_detect.c +++ b/build/detect/config/config_detect.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/build/detect/config/old/cpuid_x86.c b/build/detect/config/old/cpuid_x86.c index 1805d9643..f4985e391 100644 --- a/build/detect/config/old/cpuid_x86.c +++ b/build/detect/config/old/cpuid_x86.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2015, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/build/irun.py b/build/irun.py index d9d1e6b77..429981603 100755 --- a/build/irun.py +++ b/build/irun.py @@ -5,7 +5,7 @@ # libraries. # # Copyright (C) 2018, The University of Texas at Austin -# Copyright (C) 2018, Advanced Micro Devices, Inc. +# Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are diff --git a/config/zen/amd_config.mk b/config/zen/amd_config.mk new file mode 100644 index 000000000..658cee4c6 --- /dev/null +++ b/config/zen/amd_config.mk @@ -0,0 +1,83 @@ +# +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2019, Advanced Micro Devices, Inc. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name(s) of the copyright holder(s) nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# All the common flags for AMD architectures will be added here + +# NOTE: The build system will append these variables with various +# general-purpose/configuration-agnostic flags in common.mk. You +# may specify additional flags here as needed. +CPPROCFLAGS := +CMISCFLAGS := +CPICFLAGS := +CWARNFLAGS := + +ifneq ($(DEBUG_TYPE),off) +CDBGFLAGS := -g +endif + +ifeq ($(DEBUG_TYPE),noopt) +COPTFLAGS := -O0 +else +COPTFLAGS := -O3 -fomit-frame-pointer +endif + +# Flags specific to optimized kernels. +CKOPTFLAGS := $(COPTFLAGS) +ifeq ($(CC_VENDOR),gcc) +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma +else +ifeq ($(CC_VENDOR),clang) +#CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma +# When compiling with AOCC, add these flags to the default flags set above. +ifeq ($(strip $(shell clang -v |& head -1 | grep -c 'AOCC.LLVM.2.0.0')),1) +CKVECFLAGS += -mllvm -disable-licm-vrp +endif +else +$(error gcc or clang are required for this configuration.) +endif +endif + +# Flags specific to reference kernels. +CROPTFLAGS := $(CKOPTFLAGS) +ifeq ($(CC_VENDOR),gcc) +CRVECFLAGS := $(CKVECFLAGS) -funsafe-math-optimizations -ffp-contract=fast +else +ifeq ($(CC_VENDOR),clang) +CRVECFLAGS := $(CKVECFLAGS) -funsafe-math-optimizations -ffp-contract=fast +else +CRVECFLAGS := $(CKVECFLAGS) +endif +endif + diff --git a/config/zen/bli_cntx_init_zen.c b/config/zen/bli_cntx_init_zen.c index a61e0e51d..76b03b8a7 100644 --- a/config/zen/bli_cntx_init_zen.c +++ b/config/zen/bli_cntx_init_zen.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -115,15 +115,37 @@ void bli_cntx_init_zen( cntx_t* cntx ) // s d c z bli_blksz_init_easy( &blkszs[ BLIS_MR ], 6, 6, 3, 3 ); bli_blksz_init_easy( &blkszs[ BLIS_NR ], 16, 8, 8, 4 ); + +/* + Multi Instance performance improvement of DGEMM when binded to a CCX + In Multi instance each thread runs a sequential DGEMM. + + a) If BLIS is run in a multi-instance mode with + CPU freq 2.6/2.2 Ghz + DDR4 clock frequency 2400Mhz + mc = 240, kc = 512, and nc = 2040 + has better performance on EPYC server, over the default block sizes. + + b) If BLIS is run in Single Instance mode + mc = 510, kc = 1024 and nc = 4080 +*/ + #ifdef BLIS_ENABLE_ZEN_BLOCK_SIZES // Zen optmized level 3 cache block sizes + #if BLIS_ENABLE_SINGLE_INSTANCE_BLOCK_SIZES bli_blksz_init_easy( &blkszs[ BLIS_MC ], 1020, 510, 510, 255 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 1024, 1024, 1024, 1024 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 8060, 4080, 4080, 3056 ); + #else + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 240, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 2040, 1528 ); + #endif #else bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 8060, 4080, 4080, 3056 ); #endif - bli_blksz_init_easy( &blkszs[ BLIS_NC ], 8160, 4080, 4080, 3056 ); bli_blksz_init_easy( &blkszs[ BLIS_AF ], 8, 8, -1, -1 ); bli_blksz_init_easy( &blkszs[ BLIS_DF ], 8, 8, -1, -1 ); diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index 2e3f5518e..3f41a53bb 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -43,13 +43,27 @@ #define BLIS_THREAD_MAX_JR 1 #define BLIS_ENABLE_ZEN_BLOCK_SIZES -//#define BLIS_ENABLE_SMALL_MATRIX +#define BLIS_ENABLE_SMALL_MATRIX +#define BLIS_ENABLE_SMALL_MATRIX_TRSM + // This will select the threshold below which small matrix code will be called. #define BLIS_SMALL_MATRIX_THRES 700 #define BLIS_SMALL_M_RECT_MATRIX_THRES 160 #define BLIS_SMALL_K_RECT_MATRIX_THRES 128 +#define BLIS_SMALL_MATRIX_THRES_TRSM 32768 //128(128+128) => m*(m+n) +#define BLIS_SMALL_MATRIX_A_THRES_TRSM 128 +#define BLIS_SMALL_MATRIX_A_THRES_M_SYRK 96 +#define BLIS_SMALL_MATRIX_A_THRES_N_SYRK 128 + +//This macro will enable BLIS DGEMM to choose block sizes for a single instance mode +#define BLIS_ENABLE_SINGLE_INSTANCE_BLOCK_SIZES 0 + +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES 250 +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_ALXB_NAPLES 90 + +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO 22 //#endif diff --git a/config/zen/make_defs.mk b/config/zen/make_defs.mk index 1b9db5371..8f975d5bc 100644 --- a/config/zen/make_defs.mk +++ b/config/zen/make_defs.mk @@ -5,6 +5,7 @@ # libraries. # # Copyright (C) 2014, The University of Texas at Austin +# Copyright (C) 2019, Advanced Micro Devices, Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -32,6 +33,9 @@ # # +# FLAGS that are specific to the 'zen' architecture are added here. +# FLAGS that are common for all the AMD architectures are present in +# amd_config.mk. # Declare the name of the current configuration and add it to the # running list of configurations included by common.mk. @@ -42,56 +46,38 @@ THIS_CONFIG := zen # --- Determine the C compiler and related flags --- # -# NOTE: The build system will append these variables with various -# general-purpose/configuration-agnostic flags in common.mk. You -# may specify additional flags here as needed. -CPPROCFLAGS := -CMISCFLAGS := -CPICFLAGS := -CWARNFLAGS := +# Include the file containing common flags for all AMD architectures. +AMD_CONFIG_FILE := amd_config.mk +AMD_CONFIG_PATH := $(BASE_SHARE_PATH)/config/zen +-include $(AMD_CONFIG_PATH)/$(AMD_CONFIG_FILE) -ifneq ($(DEBUG_TYPE),off) -CDBGFLAGS := -g -endif - -ifeq ($(DEBUG_TYPE),noopt) -COPTFLAGS := -O0 -else -COPTFLAGS := -O3 -endif - -# Flags specific to optimized kernels. -CKOPTFLAGS := $(COPTFLAGS) ifeq ($(CC_VENDOR),gcc) -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -ifeq ($(GCC_OT_6_1_0),yes) # If gcc is older than 6.1.0, we must use -march=bdver4 and then remove the # Bulldozer instruction sets that were omitted from Zen. # Additionally, if gcc is 4.9 (clang 3.5?) or newer, we may want to add # Zen-specific instructions back into the mix: # -mclzero -madx -mrdseed -mmwaitx -msha -mxsavec -mxsaves -mclflushopt -mpopcnt -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +ifeq ($(GCC_OT_6_1_0),yes) +CRVECFLAGS += -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +CKVECFLAGS += -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +else +# If gcc is at least 6.1.0, then we can specify the microarchitecture using +# the preferred option. +CRVECFLAGS += -march=znver1 +CKVECFLAGS += -march=znver1 endif else ifeq ($(CC_VENDOR),clang) -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +# I couldn't find which versions of clang added support for -march=znver1, +# so we don't even bother attempting the differentiation that appears in the +# gcc branch above. +CRVECFLAGS += -march=znver1 +CKVECFLAGS += -march=znver1 else $(error gcc or clang are required for this configuration.) endif endif -# Flags specific to reference kernels. -CROPTFLAGS := $(CKOPTFLAGS) -ifeq ($(CC_VENDOR),gcc) -CRVECFLAGS := $(CKVECFLAGS) -funsafe-math-optimizations -ffp-contract=fast -else -ifeq ($(CC_VENDOR),clang) -CRVECFLAGS := $(CKVECFLAGS) -funsafe-math-optimizations -ffp-contract=fast -else -CRVECFLAGS := $(CKVECFLAGS) -endif -endif - # Store all of the variables here to new variables containing the # configuration name. $(eval $(call store-make-defs,$(THIS_CONFIG))) diff --git a/config/zen/old/bli_kernel.h b/config/zen/old/bli_kernel.h index 68b9e88e0..cd324fd9a 100644 --- a/config/zen/old/bli_kernel.h +++ b/config/zen/old/bli_kernel.h @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/config/zen2/bli_cntx_init_zen2.c b/config/zen2/bli_cntx_init_zen2.c new file mode 100644 index 000000000..2a8af0ba0 --- /dev/null +++ b/config/zen2/bli_cntx_init_zen2.c @@ -0,0 +1,139 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2019, Advanced Micro Devices, Inc. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_cntx_init_zen2( cntx_t* cntx ) +{ + blksz_t blkszs[ BLIS_NUM_BLKSZS ]; + + // Set default kernel blocksizes and functions. + bli_cntx_init_zen2_ref( cntx ); + + // ------------------------------------------------------------------------- + + // Update the context with optimized native gemm micro-kernels and + // their storage preferences. + bli_cntx_set_l3_nat_ukrs + ( + 8, + // gemm + BLIS_GEMM_UKR, BLIS_FLOAT, bli_sgemm_haswell_asm_6x16, TRUE, + BLIS_GEMM_UKR, BLIS_DOUBLE, bli_dgemm_haswell_asm_6x8, TRUE, + BLIS_GEMM_UKR, BLIS_SCOMPLEX, bli_cgemm_haswell_asm_3x8, TRUE, + BLIS_GEMM_UKR, BLIS_DCOMPLEX, bli_zgemm_haswell_asm_3x4, TRUE, + // gemmtrsm_l + BLIS_GEMMTRSM_L_UKR, BLIS_FLOAT, bli_sgemmtrsm_l_haswell_asm_6x16, TRUE, + BLIS_GEMMTRSM_L_UKR, BLIS_DOUBLE, bli_dgemmtrsm_l_haswell_asm_6x8, TRUE, + // gemmtrsm_u + BLIS_GEMMTRSM_U_UKR, BLIS_FLOAT, bli_sgemmtrsm_u_haswell_asm_6x16, TRUE, + BLIS_GEMMTRSM_U_UKR, BLIS_DOUBLE, bli_dgemmtrsm_u_haswell_asm_6x8, TRUE, + cntx + ); + + // Update the context with optimized level-1f kernels. + bli_cntx_set_l1f_kers + ( + 4, + // axpyf + BLIS_AXPYF_KER, BLIS_FLOAT, bli_saxpyf_zen_int_8, + BLIS_AXPYF_KER, BLIS_DOUBLE, bli_daxpyf_zen_int_8, + // dotxf + BLIS_DOTXF_KER, BLIS_FLOAT, bli_sdotxf_zen_int_8, + BLIS_DOTXF_KER, BLIS_DOUBLE, bli_ddotxf_zen_int_8, + cntx + ); + + // Update the context with optimized level-1v kernels. + bli_cntx_set_l1v_kers + ( + 10, + // amaxv + BLIS_AMAXV_KER, BLIS_FLOAT, bli_samaxv_zen_int, + BLIS_AMAXV_KER, BLIS_DOUBLE, bli_damaxv_zen_int, + // axpyv + + BLIS_AXPYV_KER, BLIS_FLOAT, bli_saxpyv_zen_int10, + BLIS_AXPYV_KER, BLIS_DOUBLE, bli_daxpyv_zen_int10, + + // dotv + BLIS_DOTV_KER, BLIS_FLOAT, bli_sdotv_zen_int, + BLIS_DOTV_KER, BLIS_DOUBLE, bli_ddotv_zen_int, + // dotxv + BLIS_DOTXV_KER, BLIS_FLOAT, bli_sdotxv_zen_int, + BLIS_DOTXV_KER, BLIS_DOUBLE, bli_ddotxv_zen_int, + // scalv + + BLIS_SCALV_KER, BLIS_FLOAT, bli_sscalv_zen_int10, + BLIS_SCALV_KER, BLIS_DOUBLE, bli_dscalv_zen_int10, + + cntx + ); + + // Initialize level-3 blocksize objects with architecture-specific values. + // s d c z + bli_blksz_init_easy( &blkszs[ BLIS_MR ], 6, 6, 3, 3 ); + bli_blksz_init_easy( &blkszs[ BLIS_NR ], 16, 8, 8, 4 ); +#if AOCL_BLIS_MULTIINSTANCE + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 240, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); +#else + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); +#endif + + bli_blksz_init_easy( &blkszs[ BLIS_AF ], 8, 8, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_DF ], 8, 8, -1, -1 ); + + // Update the context with the current architecture's register and cache + // blocksizes (and multiples) for native execution. + bli_cntx_set_blkszs + ( + BLIS_NAT, 7, + // level-3 + BLIS_NC, &blkszs[ BLIS_NC ], BLIS_NR, + BLIS_KC, &blkszs[ BLIS_KC ], BLIS_KR, + BLIS_MC, &blkszs[ BLIS_MC ], BLIS_MR, + BLIS_NR, &blkszs[ BLIS_NR ], BLIS_NR, + BLIS_MR, &blkszs[ BLIS_MR ], BLIS_MR, + // level-1f + BLIS_AF, &blkszs[ BLIS_AF ], BLIS_AF, + BLIS_DF, &blkszs[ BLIS_DF ], BLIS_DF, + cntx + ); +} + diff --git a/config/zen2/bli_family_zen2.h b/config/zen2/bli_family_zen2.h new file mode 100644 index 000000000..def0e21f9 --- /dev/null +++ b/config/zen2/bli_family_zen2.h @@ -0,0 +1,74 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2019, Advanced Micro Devices, Inc + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef BLI_FAMILY_ZEN2_ +#define BLI_FAMILY_ZEN2_ + +// By default, it is effective to parallelize the outer loops. +// Setting these macros to 1 will force JR and IR inner loops +// to be not paralleized. +#define BLIS_THREAD_MAX_IR 1 +#define BLIS_THREAD_MAX_JR 1 + + +#define BLIS_ENABLE_SMALL_MATRIX +#define BLIS_ENABLE_SMALL_MATRIX_TRSM + + +// This will select the threshold below which small matrix code will be called. +#define BLIS_SMALL_MATRIX_THRES 700 +#define BLIS_SMALL_M_RECT_MATRIX_THRES 160 +#define BLIS_SMALL_K_RECT_MATRIX_THRES 128 + +#define BLIS_SMALL_MATRIX_THRES_TRSM 32768 //128(128+128) => m*(m+n) +#define BLIS_SMALL_MATRIX_A_THRES_TRSM 128 +#define BLIS_SMALL_MATRIX_A_THRES_M_SYRK 96 +#define BLIS_SMALL_MATRIX_A_THRES_N_SYRK 128 + +#define BLIS_ENABLE_SMALL_MATRIX_ROME +#define BLIS_SMALL_MATRIX_THRES_ROME 400 + +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME 120 +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_ALXB_ROME 60 +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_XAUB_ROME 150 + +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO 22 + +// When running HPL with pure MPI without DGEMM threading (Single-threaded +// BLIS), defining this macro as 1 yields better performance. +#define AOCL_BLIS_MULTIINSTANCE 0 + +#endif + diff --git a/config/zen2/make_defs.mk b/config/zen2/make_defs.mk new file mode 100644 index 000000000..f072f1293 --- /dev/null +++ b/config/zen2/make_defs.mk @@ -0,0 +1,88 @@ +# +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2014, The University of Texas at Austin +# Copyright (C) 2019, Advanced Micro Devices, Inc. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name(s) of the copyright holder(s) nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# FLAGS that are specific to the 'zen2' architecture are added here. +# FLAGS that are common for all the AMD architectures are present in +# config/zen/amd_config.mk. + +# Declare the name of the current configuration and add it to the +# running list of configurations included by common.mk. +THIS_CONFIG := zen2 +#CONFIGS_INCL += $(THIS_CONFIG) + +# +# --- Determine the C compiler and related flags --- +# + +# Include file containing common flags for all AMD architectures. +AMD_CONFIG_FILE := amd_config.mk +AMD_CONFIG_PATH := $(BASE_SHARE_PATH)/config/zen +-include $(AMD_CONFIG_PATH)/$(AMD_CONFIG_FILE) + +ifeq ($(CC_VENDOR),gcc) +ifeq ($(GCC_OT_9_1_0),yes) +ifeq ($(GCC_OT_6_1_0),yes) +# If gcc is older than 6.1.0, we must use -march=bdver4 and then remove the +# Bulldozer instruction sets that were omitted from Zen. +CRVECFLAGS += -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +CKVECFLAGS += -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +else +# If gcc is older than 9.1.0 but at least 6.1.0, then we can use -march=znver1 +# as the fallback option. +CRVECFLAGS += -march=znver1 -mno-avx256-split-unaligned-store +CKVECFLAGS += -march=znver1 -mno-avx256-split-unaligned-store +endif +else +# If gcc is at least 9.1.0, then we can specify the microarchitecture using +# the preferred option. +CRVECFLAGS += -march=znver2 +CKVECFLAGS += -march=znver2 +endif +else +ifeq ($(CC_VENDOR),clang) +# I couldn't find which versions of clang added support for -march=znver1 +# or -march=znver2, so we don't even bother attempting the differentiation +# that appears in the gcc branch above. +CRVECFLAGS += -march=znver2 +CKVECFLAGS += -march=znver2 +else +$(error gcc or clang are required for this configuration.) +endif +endif + +# Store all of the variables here to new variables containing the +# configuration name. +$(eval $(call store-make-defs,$(THIS_CONFIG))) + diff --git a/config_registry b/config_registry index 0680db083..ed859cd3c 100644 --- a/config_registry +++ b/config_registry @@ -10,7 +10,7 @@ # Processor families. x86_64: intel64 amd64 intel64: skx knl haswell sandybridge penryn generic -amd64: zen excavator steamroller piledriver bulldozer generic +amd64: zen2 zen excavator steamroller piledriver bulldozer generic # NOTE: ARM families will remain disabled until runtime hardware detection # logic is added to BLIS. #arm64: cortexa57 generic @@ -24,6 +24,7 @@ sandybridge: sandybridge penryn: penryn # AMD architectures. +zen2: zen2/zen2/zen/haswell zen: zen/zen/haswell excavator: excavator/piledriver steamroller: steamroller/piledriver diff --git a/configure b/configure index 8107e588b..13e06e4d8 100755 --- a/configure +++ b/configure @@ -5,7 +5,7 @@ # libraries. # # Copyright (C) 2014, The University of Texas at Austin -# Copyright (C) 2019, Advanced Micro Devices, Inc. +# Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -1516,19 +1516,37 @@ check_compiler_version_ranges() # comments: # These older versions of gcc do not explicitly support the Zen (Zen1) # microarchitecture; the newest microarchitectural value understood by - # these versions is '-march=bdver4' [3]. However, support for them can - # be attained in a roundabout way by starting with the instruction sets - # enabled by '-march=bdver4' and then disabling the instruction sets - # that were removed in the transition from Excavator to Zen, namely: - # FMA4, TBM, XOP, and LWP. Newer versions of gcc support Zen via the - # '-march=znver1' option [4]. + # these versions is '-march=bdver4' [3]. However, basic support for these + # older versions can be attained in a roundabout way by starting with the + # instruction sets enabled by '-march=bdver4' and then disabling the + # instruction sets that were removed in the transition from Excavator to + # Zen, namely: FMA4, TBM, XOP, and LWP. Newer versions of gcc support Zen + # via the '-march=znver1' option [4]. # # [3] https://gcc.gnu.org/onlinedocs/gcc-5.5.0/gcc/x86-Options.html#x86-Options # [4] https://gcc.gnu.org/onlinedocs/gcc-6.1.0/gcc/x86-Options.html#x86-Options # + # range: gcc < 9.1 (ie: 8.3 or older) + # variable: gcc_older_than_9_1_0 + # comments: + # These older versions of gcc do not explicitly support the Zen2 + # microarchitecture; the newest microarchitectural value understood by + # these versions is either '-march=znver1' (if !gcc_older_than_6_1_0) [5] + # or '-march=bdver4' (if gcc_older_than_6_1_0) [3]. If gcc is 6.1 or + # newer, '-march=znver1' may be used (since the instruction sets it + # enables are a subset of those enabled by '-march=znver2'); otherwise, + # '-march=bdver4' must be used in conjuction with disabling the + # instruction sets that were removed in the transition from Excavator to + # Zen, as described in the section above for gcc_older_than_6_1_0. + # Newer versions of gcc support Zen2 via the '-march=znver2' option [6]. + # + # [5] https://gcc.gnu.org/onlinedocs/gcc-8.3.0/gcc/x86-Options.html#x86-Options + # [6] https://gcc.gnu.org/onlinedocs/gcc-9.1.0/gcc/x86-Options.html#x86-Options + # gcc_older_than_4_9_0='no' gcc_older_than_6_1_0='no' + gcc_older_than_9_1_0='no' echo "${script_name}: checking ${cc} ${cc_version} against known consequential version ranges." @@ -1548,6 +1566,12 @@ check_compiler_version_ranges() echo "${script_name}: note: found ${cc} version older than 6.1." gcc_older_than_6_1_0='yes' fi + + # Check for gcc < 9.1.0 (ie: 8.3 or older). + if [ ${cc_major} -lt 9 ]; then + echo "${script_name}: note: found ${cc} version older than 9.1." + gcc_older_than_9_1_0='yes' + fi fi # icc @@ -3037,6 +3061,7 @@ main() | sed -e "s/@CC_VENDOR@/${cc_vendor}/g" \ | sed -e "s/@gcc_older_than_4_9_0@/${gcc_older_than_4_9_0}/g" \ | sed -e "s/@gcc_older_than_6_1_0@/${gcc_older_than_6_1_0}/g" \ + | sed -e "s/@gcc_older_than_9_1_0@/${gcc_older_than_9_1_0}/g" \ | sed -e "s/@CC@/${cc_esc}/g" \ | sed -e "s/@CXX@/${cxx_esc}/g" \ | sed -e "s/@RANLIB@/${ranlib_esc}/g" \ diff --git a/frame/1m/packm/bli_packm.h b/frame/1m/packm/bli_packm.h index fbc02e392..85f701165 100644 --- a/frame/1m/packm/bli_packm.h +++ b/frame/1m/packm/bli_packm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm_blk_var1.c b/frame/1m/packm/bli_packm_blk_var1.c index 10c4ad93d..0c19829f2 100644 --- a/frame/1m/packm/bli_packm_blk_var1.c +++ b/frame/1m/packm/bli_packm_blk_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm_cntl.c b/frame/1m/packm/bli_packm_cntl.c index c1188980b..5b3e26421 100644 --- a/frame/1m/packm/bli_packm_cntl.c +++ b/frame/1m/packm/bli_packm_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm_cntl.h b/frame/1m/packm/bli_packm_cntl.h index 5f25ebcee..5ce1801b9 100644 --- a/frame/1m/packm/bli_packm_cntl.h +++ b/frame/1m/packm/bli_packm_cntl.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm_thrinfo.c b/frame/1m/packm/bli_packm_thrinfo.c index 92162c422..4b57971ef 100644 --- a/frame/1m/packm/bli_packm_thrinfo.c +++ b/frame/1m/packm/bli_packm_thrinfo.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm_thrinfo.h b/frame/1m/packm/bli_packm_thrinfo.h index 7d35cbc93..85b61931c 100644 --- a/frame/1m/packm/bli_packm_thrinfo.h +++ b/frame/1m/packm/bli_packm_thrinfo.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm_var.h b/frame/1m/packm/bli_packm_var.h index f400c0550..97aa875bd 100644 --- a/frame/1m/packm/bli_packm_var.h +++ b/frame/1m/packm/bli_packm_var.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/unpackm/bli_unpackm_cntl.c b/frame/1m/unpackm/bli_unpackm_cntl.c index 78247e04e..f2be05a54 100644 --- a/frame/1m/unpackm/bli_unpackm_cntl.c +++ b/frame/1m/unpackm/bli_unpackm_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/unpackm/bli_unpackm_cntl.h b/frame/1m/unpackm/bli_unpackm_cntl.h index 6488cd945..5c41d9465 100644 --- a/frame/1m/unpackm/bli_unpackm_cntl.h +++ b/frame/1m/unpackm/bli_unpackm_cntl.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/bli_l3_cntl.c b/frame/3/bli_l3_cntl.c index efdca53db..f6bfbedbb 100644 --- a/frame/3/bli_l3_cntl.c +++ b/frame/3/bli_l3_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/bli_l3_cntl.h b/frame/3/bli_l3_cntl.h index 0c04f348c..c308c8a96 100644 --- a/frame/3/bli_l3_cntl.h +++ b/frame/3/bli_l3_cntl.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/bli_l3_packm.c b/frame/3/bli_l3_packm.c index bfb066bfb..b5dc17d4a 100644 --- a/frame/3/bli_l3_packm.c +++ b/frame/3/bli_l3_packm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/bli_l3_packm.h b/frame/3/bli_l3_packm.h index 37b1db105..696dabf59 100644 --- a/frame/3/bli_l3_packm.h +++ b/frame/3/bli_l3_packm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/bli_l3_thrinfo.c b/frame/3/bli_l3_thrinfo.c index 4f073cb20..0eaf10840 100644 --- a/frame/3/bli_l3_thrinfo.c +++ b/frame/3/bli_l3_thrinfo.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/bli_l3_thrinfo.h b/frame/3/bli_l3_thrinfo.h index 15d8faed6..1dc785615 100644 --- a/frame/3/bli_l3_thrinfo.h +++ b/frame/3/bli_l3_thrinfo.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_blk_var1.c b/frame/3/gemm/bli_gemm_blk_var1.c index b53711901..3b7634338 100644 --- a/frame/3/gemm/bli_gemm_blk_var1.c +++ b/frame/3/gemm/bli_gemm_blk_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_blk_var2.c b/frame/3/gemm/bli_gemm_blk_var2.c index cd5a833f6..d89a71053 100644 --- a/frame/3/gemm/bli_gemm_blk_var2.c +++ b/frame/3/gemm/bli_gemm_blk_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_cntl.c b/frame/3/gemm/bli_gemm_cntl.c index 3a6f6730c..d7cd0a92c 100644 --- a/frame/3/gemm/bli_gemm_cntl.c +++ b/frame/3/gemm/bli_gemm_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_cntl.h b/frame/3/gemm/bli_gemm_cntl.h index 26b53459e..bff91b58a 100644 --- a/frame/3/gemm/bli_gemm_cntl.h +++ b/frame/3/gemm/bli_gemm_cntl.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_front.c b/frame/3/gemm/bli_gemm_front.c index 40389b97e..26a8dc58c 100644 --- a/frame/3/gemm/bli_gemm_front.c +++ b/frame/3/gemm/bli_gemm_front.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -57,7 +57,8 @@ void bli_gemm_front #ifdef BLIS_ENABLE_SMALL_MATRIX // Only handle small problems separately for homogeneous datatypes. if ( bli_obj_dt( a ) == bli_obj_dt( b ) && - bli_obj_dt( a ) == bli_obj_dt( c ) ) + bli_obj_dt( a ) == bli_obj_dt( c ) && + bli_obj_comp_prec( c ) == bli_obj_prec( c ) ) { gint_t status = bli_gemm_small( alpha, a, b, beta, c, cntx, cntl ); if ( status == BLIS_SUCCESS ) return; diff --git a/frame/3/gemm/bli_gemm_front.h b/frame/3/gemm/bli_gemm_front.h index ba65bab8d..2728ce8f7 100644 --- a/frame/3/gemm/bli_gemm_front.h +++ b/frame/3/gemm/bli_gemm_front.h @@ -44,6 +44,7 @@ void bli_gemm_front cntl_t* cntl ); +#ifdef BLIS_ENABLE_SMALL_MATRIX err_t bli_gemm_small ( obj_t* alpha, @@ -54,3 +55,5 @@ err_t bli_gemm_small cntx_t* cntx, cntl_t* cntl ); +#endif + diff --git a/frame/3/gemm/bli_gemm_int.c b/frame/3/gemm/bli_gemm_int.c index 25a6215df..5128bea69 100644 --- a/frame/3/gemm/bli_gemm_int.c +++ b/frame/3/gemm/bli_gemm_int.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_ker_var1.c b/frame/3/gemm/bli_gemm_ker_var1.c index 4dcffd279..096091e76 100644 --- a/frame/3/gemm/bli_gemm_ker_var1.c +++ b/frame/3/gemm/bli_gemm_ker_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_ker_var2.c b/frame/3/gemm/bli_gemm_ker_var2.c index 41bb3f455..e9b43f2ac 100644 --- a/frame/3/gemm/bli_gemm_ker_var2.c +++ b/frame/3/gemm/bli_gemm_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_md.c b/frame/3/gemm/bli_gemm_md.c index b08bea2c4..1f5a8e593 100644 --- a/frame/3/gemm/bli_gemm_md.c +++ b/frame/3/gemm/bli_gemm_md.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_var.h b/frame/3/gemm/bli_gemm_var.h index 34cf95ae6..b08271e9b 100644 --- a/frame/3/gemm/bli_gemm_var.h +++ b/frame/3/gemm/bli_gemm_var.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/ind/bli_gemm4mb_ker_var2.c b/frame/3/gemm/ind/bli_gemm4mb_ker_var2.c index e4b377b37..cbc3a1929 100644 --- a/frame/3/gemm/ind/bli_gemm4mb_ker_var2.c +++ b/frame/3/gemm/ind/bli_gemm4mb_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/other/bli_gemm_ker_var2.c b/frame/3/gemm/other/bli_gemm_ker_var2.c index 6ae8df0c1..73ca61181 100644 --- a/frame/3/gemm/other/bli_gemm_ker_var2.c +++ b/frame/3/gemm/other/bli_gemm_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/other/bli_gemm_ker_var2rr.c b/frame/3/gemm/other/bli_gemm_ker_var2rr.c index a213e50fc..f79f1311e 100644 --- a/frame/3/gemm/other/bli_gemm_ker_var2rr.c +++ b/frame/3/gemm/other/bli_gemm_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/other/bli_gemm_ker_var2sl.c b/frame/3/gemm/other/bli_gemm_ker_var2sl.c index 0d710bd73..48ef2791f 100644 --- a/frame/3/gemm/other/bli_gemm_ker_var2sl.c +++ b/frame/3/gemm/other/bli_gemm_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/bli_herk_l_ker_var2.c b/frame/3/herk/bli_herk_l_ker_var2.c index d077b8f89..d09424d9a 100644 --- a/frame/3/herk/bli_herk_l_ker_var2.c +++ b/frame/3/herk/bli_herk_l_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/bli_herk_u_ker_var2.c b/frame/3/herk/bli_herk_u_ker_var2.c index b20a96df7..51fd5d4d6 100644 --- a/frame/3/herk/bli_herk_u_ker_var2.c +++ b/frame/3/herk/bli_herk_u_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/bli_herk_var.h b/frame/3/herk/bli_herk_var.h index 3c565e1b0..00b85fc5c 100644 --- a/frame/3/herk/bli_herk_var.h +++ b/frame/3/herk/bli_herk_var.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/bli_herk_x_ker_var2.c b/frame/3/herk/bli_herk_x_ker_var2.c index c0ce23255..fb4048cd2 100644 --- a/frame/3/herk/bli_herk_x_ker_var2.c +++ b/frame/3/herk/bli_herk_x_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_l_ker_var2.1looprr.c b/frame/3/herk/other/bli_herk_l_ker_var2.1looprr.c index 38675b11b..7238716a7 100644 --- a/frame/3/herk/other/bli_herk_l_ker_var2.1looprr.c +++ b/frame/3/herk/other/bli_herk_l_ker_var2.1looprr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_l_ker_var2.c b/frame/3/herk/other/bli_herk_l_ker_var2.c index 904da9f5e..650acf0d7 100644 --- a/frame/3/herk/other/bli_herk_l_ker_var2.c +++ b/frame/3/herk/other/bli_herk_l_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_l_ker_var2rr.c b/frame/3/herk/other/bli_herk_l_ker_var2rr.c index a313f04b2..19edea32b 100644 --- a/frame/3/herk/other/bli_herk_l_ker_var2rr.c +++ b/frame/3/herk/other/bli_herk_l_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_l_ker_var2sl.c b/frame/3/herk/other/bli_herk_l_ker_var2sl.c index f913cced2..68e9de181 100644 --- a/frame/3/herk/other/bli_herk_l_ker_var2sl.c +++ b/frame/3/herk/other/bli_herk_l_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_u_ker_var2.1looprr.c b/frame/3/herk/other/bli_herk_u_ker_var2.1looprr.c index cd4a4e7ad..f1b191a7e 100644 --- a/frame/3/herk/other/bli_herk_u_ker_var2.1looprr.c +++ b/frame/3/herk/other/bli_herk_u_ker_var2.1looprr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_u_ker_var2.c b/frame/3/herk/other/bli_herk_u_ker_var2.c index 0bdc0b0a4..7fbc90fed 100644 --- a/frame/3/herk/other/bli_herk_u_ker_var2.c +++ b/frame/3/herk/other/bli_herk_u_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_u_ker_var2rr.c b/frame/3/herk/other/bli_herk_u_ker_var2rr.c index 4ffa8085c..1643bbc39 100644 --- a/frame/3/herk/other/bli_herk_u_ker_var2rr.c +++ b/frame/3/herk/other/bli_herk_u_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_u_ker_var2sl.c b/frame/3/herk/other/bli_herk_u_ker_var2sl.c index 7af7ee56d..24db2a4a3 100644 --- a/frame/3/herk/other/bli_herk_u_ker_var2sl.c +++ b/frame/3/herk/other/bli_herk_u_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/syrk/bli_syrk_front.c b/frame/3/syrk/bli_syrk_front.c index 534848e33..b9dad3c38 100644 --- a/frame/3/syrk/bli_syrk_front.c +++ b/frame/3/syrk/bli_syrk_front.c @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -46,7 +47,6 @@ void bli_syrk_front ) { bli_init_once(); - obj_t a_local; obj_t at_local; obj_t c_local; @@ -71,6 +71,12 @@ void bli_syrk_front bli_obj_alias_to( a, &at_local ); bli_obj_induce_trans( &at_local ); +#ifdef BLIS_ENABLE_SMALL_MATRIX + gint_t status = bli_syrk_small( alpha, &a_local, &at_local, beta, &c_local, + cntx, cntl ); + if ( status == BLIS_SUCCESS ) return; +#endif + // An optimization: If C is stored by rows and the micro-kernel prefers // contiguous columns, or if C is stored by columns and the micro-kernel // prefers contiguous rows, transpose the entire operation to allow the diff --git a/frame/3/syrk/bli_syrk_front.h b/frame/3/syrk/bli_syrk_front.h index 28d1e13f6..bf8d26a52 100644 --- a/frame/3/syrk/bli_syrk_front.h +++ b/frame/3/syrk/bli_syrk_front.h @@ -42,3 +42,17 @@ void bli_syrk_front rntm_t* rntm, cntl_t* cntl ); + +#ifdef BLIS_ENABLE_SMALL_MATRIX +err_t bli_syrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); +#endif + diff --git a/frame/3/trmm/bli_trmm_front.c b/frame/3/trmm/bli_trmm_front.c index aee9d1d6f..fa9a6389d 100644 --- a/frame/3/trmm/bli_trmm_front.c +++ b/frame/3/trmm/bli_trmm_front.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_ll_ker_var2.c b/frame/3/trmm/bli_trmm_ll_ker_var2.c index 98e62926c..5c7a7bcd0 100644 --- a/frame/3/trmm/bli_trmm_ll_ker_var2.c +++ b/frame/3/trmm/bli_trmm_ll_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_lu_ker_var2.c b/frame/3/trmm/bli_trmm_lu_ker_var2.c index 624604141..06705da44 100644 --- a/frame/3/trmm/bli_trmm_lu_ker_var2.c +++ b/frame/3/trmm/bli_trmm_lu_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_rl_ker_var2.c b/frame/3/trmm/bli_trmm_rl_ker_var2.c index 117cf63c5..9a17ed644 100644 --- a/frame/3/trmm/bli_trmm_rl_ker_var2.c +++ b/frame/3/trmm/bli_trmm_rl_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_ru_ker_var2.c b/frame/3/trmm/bli_trmm_ru_ker_var2.c index ea59959c7..1062b95d8 100644 --- a/frame/3/trmm/bli_trmm_ru_ker_var2.c +++ b/frame/3/trmm/bli_trmm_ru_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_var.h b/frame/3/trmm/bli_trmm_var.h index 09694ca5c..262b0490f 100644 --- a/frame/3/trmm/bli_trmm_var.h +++ b/frame/3/trmm/bli_trmm_var.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_xx_ker_var2.c b/frame/3/trmm/bli_trmm_xx_ker_var2.c index 343aaa078..330d06ae8 100644 --- a/frame/3/trmm/bli_trmm_xx_ker_var2.c +++ b/frame/3/trmm/bli_trmm_xx_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_ll_ker_var2.c b/frame/3/trmm/other/bli_trmm_ll_ker_var2.c index 3747a0dcf..1e5f9a9f4 100644 --- a/frame/3/trmm/other/bli_trmm_ll_ker_var2.c +++ b/frame/3/trmm/other/bli_trmm_ll_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_ll_ker_var2rr.c b/frame/3/trmm/other/bli_trmm_ll_ker_var2rr.c index ea979d7c3..3b2575f66 100644 --- a/frame/3/trmm/other/bli_trmm_ll_ker_var2rr.c +++ b/frame/3/trmm/other/bli_trmm_ll_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_ll_ker_var2sl.c b/frame/3/trmm/other/bli_trmm_ll_ker_var2sl.c index e612b340c..1dbd0053a 100644 --- a/frame/3/trmm/other/bli_trmm_ll_ker_var2sl.c +++ b/frame/3/trmm/other/bli_trmm_ll_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_lu_ker_var2.c b/frame/3/trmm/other/bli_trmm_lu_ker_var2.c index 9a4e36b65..17d2f9a2f 100644 --- a/frame/3/trmm/other/bli_trmm_lu_ker_var2.c +++ b/frame/3/trmm/other/bli_trmm_lu_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_lu_ker_var2rr.c b/frame/3/trmm/other/bli_trmm_lu_ker_var2rr.c index 551bc097d..f1a586ce6 100644 --- a/frame/3/trmm/other/bli_trmm_lu_ker_var2rr.c +++ b/frame/3/trmm/other/bli_trmm_lu_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_lu_ker_var2sl.c b/frame/3/trmm/other/bli_trmm_lu_ker_var2sl.c index 132c732d6..26806220a 100644 --- a/frame/3/trmm/other/bli_trmm_lu_ker_var2sl.c +++ b/frame/3/trmm/other/bli_trmm_lu_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_rl_ker_var2.c b/frame/3/trmm/other/bli_trmm_rl_ker_var2.c index b29df0850..d856cc383 100644 --- a/frame/3/trmm/other/bli_trmm_rl_ker_var2.c +++ b/frame/3/trmm/other/bli_trmm_rl_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_rl_ker_var2rr.c b/frame/3/trmm/other/bli_trmm_rl_ker_var2rr.c index 14b235918..80350b401 100644 --- a/frame/3/trmm/other/bli_trmm_rl_ker_var2rr.c +++ b/frame/3/trmm/other/bli_trmm_rl_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_rl_ker_var2sl.c b/frame/3/trmm/other/bli_trmm_rl_ker_var2sl.c index cf4a6e086..a44942da6 100644 --- a/frame/3/trmm/other/bli_trmm_rl_ker_var2sl.c +++ b/frame/3/trmm/other/bli_trmm_rl_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_ru_ker_var2.c b/frame/3/trmm/other/bli_trmm_ru_ker_var2.c index 602f4cc3b..120974ef0 100644 --- a/frame/3/trmm/other/bli_trmm_ru_ker_var2.c +++ b/frame/3/trmm/other/bli_trmm_ru_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_ru_ker_var2rr.c b/frame/3/trmm/other/bli_trmm_ru_ker_var2rr.c index 03eaa6ea6..45a7b5628 100644 --- a/frame/3/trmm/other/bli_trmm_ru_ker_var2rr.c +++ b/frame/3/trmm/other/bli_trmm_ru_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_ru_ker_var2sl.c b/frame/3/trmm/other/bli_trmm_ru_ker_var2sl.c index 2411a24a4..d2b4744e6 100644 --- a/frame/3/trmm/other/bli_trmm_ru_ker_var2sl.c +++ b/frame/3/trmm/other/bli_trmm_ru_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_blk_var1.c b/frame/3/trsm/bli_trsm_blk_var1.c index 1bab54d5f..6aeac4870 100644 --- a/frame/3/trsm/bli_trsm_blk_var1.c +++ b/frame/3/trsm/bli_trsm_blk_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_blk_var2.c b/frame/3/trsm/bli_trsm_blk_var2.c index c8330b801..23fd3ed4c 100644 --- a/frame/3/trsm/bli_trsm_blk_var2.c +++ b/frame/3/trsm/bli_trsm_blk_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_cntl.c b/frame/3/trsm/bli_trsm_cntl.c index 4e7ee73f7..845370448 100644 --- a/frame/3/trsm/bli_trsm_cntl.c +++ b/frame/3/trsm/bli_trsm_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_cntl.h b/frame/3/trsm/bli_trsm_cntl.h index ce4a0b9ff..7fdb1fc4f 100644 --- a/frame/3/trsm/bli_trsm_cntl.h +++ b/frame/3/trsm/bli_trsm_cntl.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_front.c b/frame/3/trsm/bli_trsm_front.c index 5093d1a4a..e7d1ad334 100644 --- a/frame/3/trsm/bli_trsm_front.c +++ b/frame/3/trsm/bli_trsm_front.c @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -51,6 +52,11 @@ void bli_trsm_front obj_t b_local; obj_t c_local; +#ifdef BLIS_ENABLE_SMALL_MATRIX_TRSM + gint_t status = bli_trsm_small( side, alpha, a, b, cntx, cntl ); + if ( status == BLIS_SUCCESS ) return; +#endif + // Check parameters. if ( bli_error_checking_is_enabled() ) bli_trsm_check( side, alpha, a, b, &BLIS_ZERO, b, cntx ); diff --git a/frame/3/trsm/bli_trsm_front.h b/frame/3/trsm/bli_trsm_front.h index 1a08b7c75..379935536 100644 --- a/frame/3/trsm/bli_trsm_front.h +++ b/frame/3/trsm/bli_trsm_front.h @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -42,3 +43,16 @@ void bli_trsm_front rntm_t* rntm, cntl_t* cntl ); + +#ifdef BLIS_ENABLE_SMALL_MATRIX +err_t bli_trsm_small + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); +#endif + diff --git a/frame/3/trsm/bli_trsm_ll_ker_var2.c b/frame/3/trsm/bli_trsm_ll_ker_var2.c index 37823d7bf..0f69057ae 100644 --- a/frame/3/trsm/bli_trsm_ll_ker_var2.c +++ b/frame/3/trsm/bli_trsm_ll_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_lu_ker_var2.c b/frame/3/trsm/bli_trsm_lu_ker_var2.c index 853bccf91..f2cb62608 100644 --- a/frame/3/trsm/bli_trsm_lu_ker_var2.c +++ b/frame/3/trsm/bli_trsm_lu_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_rl_ker_var2.c b/frame/3/trsm/bli_trsm_rl_ker_var2.c index 87e1a0b28..1508cd596 100644 --- a/frame/3/trsm/bli_trsm_rl_ker_var2.c +++ b/frame/3/trsm/bli_trsm_rl_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_ru_ker_var2.c b/frame/3/trsm/bli_trsm_ru_ker_var2.c index 71a72ea24..54288a04d 100644 --- a/frame/3/trsm/bli_trsm_ru_ker_var2.c +++ b/frame/3/trsm/bli_trsm_ru_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_var.h b/frame/3/trsm/bli_trsm_var.h index 0f5f42de8..de7c65936 100644 --- a/frame/3/trsm/bli_trsm_var.h +++ b/frame/3/trsm/bli_trsm_var.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_xx_ker_var2.c b/frame/3/trsm/bli_trsm_xx_ker_var2.c index dfdcf2eba..2e7577f14 100644 --- a/frame/3/trsm/bli_trsm_xx_ker_var2.c +++ b/frame/3/trsm/bli_trsm_xx_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_ll_ker_var2.c b/frame/3/trsm/other/bli_trsm_ll_ker_var2.c index 1c4b0b5c7..ebd53476d 100644 --- a/frame/3/trsm/other/bli_trsm_ll_ker_var2.c +++ b/frame/3/trsm/other/bli_trsm_ll_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_ll_ker_var2rr.c b/frame/3/trsm/other/bli_trsm_ll_ker_var2rr.c index 3891bffc0..eb7435b4a 100644 --- a/frame/3/trsm/other/bli_trsm_ll_ker_var2rr.c +++ b/frame/3/trsm/other/bli_trsm_ll_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_ll_ker_var2sl.c b/frame/3/trsm/other/bli_trsm_ll_ker_var2sl.c index 1bc2f6e42..df9235740 100644 --- a/frame/3/trsm/other/bli_trsm_ll_ker_var2sl.c +++ b/frame/3/trsm/other/bli_trsm_ll_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_lu_ker_var2.c b/frame/3/trsm/other/bli_trsm_lu_ker_var2.c index 673e1eaa3..97222ffe4 100644 --- a/frame/3/trsm/other/bli_trsm_lu_ker_var2.c +++ b/frame/3/trsm/other/bli_trsm_lu_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_lu_ker_var2rr.c b/frame/3/trsm/other/bli_trsm_lu_ker_var2rr.c index 72761ee54..107aae742 100644 --- a/frame/3/trsm/other/bli_trsm_lu_ker_var2rr.c +++ b/frame/3/trsm/other/bli_trsm_lu_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_lu_ker_var2sl.c b/frame/3/trsm/other/bli_trsm_lu_ker_var2sl.c index 491ae8198..ee18058a4 100644 --- a/frame/3/trsm/other/bli_trsm_lu_ker_var2sl.c +++ b/frame/3/trsm/other/bli_trsm_lu_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_rl_ker_var2.c b/frame/3/trsm/other/bli_trsm_rl_ker_var2.c index 3293289a1..0fff5cc11 100644 --- a/frame/3/trsm/other/bli_trsm_rl_ker_var2.c +++ b/frame/3/trsm/other/bli_trsm_rl_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_ru_ker_var2.c b/frame/3/trsm/other/bli_trsm_ru_ker_var2.c index 9726fd467..f4023ace2 100644 --- a/frame/3/trsm/other/bli_trsm_ru_ker_var2.c +++ b/frame/3/trsm/other/bli_trsm_ru_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_apool.c b/frame/base/bli_apool.c index 5dd98206e..8b8f1f725 100644 --- a/frame/base/bli_apool.c +++ b/frame/base/bli_apool.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_apool.h b/frame/base/bli_apool.h index 1f7889023..f08d89c9d 100644 --- a/frame/base/bli_apool.h +++ b/frame/base/bli_apool.h @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_arch.c b/frame/base/bli_arch.c index 524340c5f..09388da0d 100644 --- a/frame/base/bli_arch.c +++ b/frame/base/bli_arch.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018-2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -104,6 +104,9 @@ void bli_arch_set_id( void ) #endif // AMD microarchitectures. +#ifdef BLIS_FAMILY_ZEN2 + id = BLIS_ARCH_ZEN2; +#endif #ifdef BLIS_FAMILY_ZEN id = BLIS_ARCH_ZEN; #endif @@ -172,6 +175,7 @@ static char* config_name[ BLIS_NUM_ARCHS ] = "sandybridge", "penryn", + "zen2", "zen", "excavator", "steamroller", diff --git a/frame/base/bli_array.c b/frame/base/bli_array.c index 3f167056e..6232cffbb 100644 --- a/frame/base/bli_array.c +++ b/frame/base/bli_array.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_array.h b/frame/base/bli_array.h index e3070ae67..cb6db0340 100644 --- a/frame/base/bli_array.h +++ b/frame/base/bli_array.h @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_check.c b/frame/base/bli_check.c index f5b3aebec..4c7f93cf4 100644 --- a/frame/base/bli_check.c +++ b/frame/base/bli_check.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_check.h b/frame/base/bli_check.h index d94648bf7..539458406 100644 --- a/frame/base/bli_check.h +++ b/frame/base/bli_check.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_clock.c b/frame/base/bli_clock.c index bd5cd9e82..a4df37e16 100644 --- a/frame/base/bli_clock.c +++ b/frame/base/bli_clock.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_cntl.c b/frame/base/bli_cntl.c index 4915be431..0de6fbc39 100644 --- a/frame/base/bli_cntl.c +++ b/frame/base/bli_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_cntl.h b/frame/base/bli_cntl.h index 72b4185b9..998a92571 100644 --- a/frame/base/bli_cntl.h +++ b/frame/base/bli_cntl.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_cntx.c b/frame/base/bli_cntx.c index 7027ebd08..b612518b8 100644 --- a/frame/base/bli_cntx.c +++ b/frame/base/bli_cntx.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_cpuid.c b/frame/base/bli_cpuid.c index c8891f075..7385143eb 100644 --- a/frame/base/bli_cpuid.c +++ b/frame/base/bli_cpuid.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018-2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -67,6 +67,14 @@ arch_t bli_cpuid_query_id( void ) // vendor. vendor = bli_cpuid_query( &family, &model, &features ); +#if 0 + printf( "vendor = %s\n", vendor==1 ? "AMD": "INTEL" ); + printf("family = %x\n", family ); + printf( "model = %x\n", model ); + + printf( "features = %x\n", features ); +#endif + if ( vendor == VENDOR_INTEL ) { // Check for each Intel configuration that is enabled, check for that @@ -100,6 +108,10 @@ arch_t bli_cpuid_query_id( void ) // Check for each AMD configuration that is enabled, check for that // microarchitecture. We check from most recent to most dated. +#ifdef BLIS_CONFIG_ZEN2 + if ( bli_cpuid_is_zen2( family, model, features ) ) + return BLIS_ARCH_ZEN2; +#endif #ifdef BLIS_CONFIG_ZEN if ( bli_cpuid_is_zen( family, model, features ) ) return BLIS_ARCH_ZEN; @@ -228,6 +240,34 @@ bool_t bli_cpuid_is_penryn // ----------------------------------------------------------------------------- +bool_t bli_cpuid_is_zen2 + ( + uint32_t family, + uint32_t model, + uint32_t features + ) +{ + // Check for expected CPU features. + const uint32_t expected = FEATURE_AVX | + FEATURE_FMA3 | + FEATURE_AVX2; + + if ( !bli_cpuid_has_features( features, expected ) ) return FALSE; + + // All Zen2 cores have a family of 0x17. + if ( family != 0x17 ) return FALSE; + + // Finally, check for specific models: + // - 0x30-0xff (THIS NEEDS UPDATING) + const bool_t is_arch + = + ( 0x30 <= model && model <= 0xff ); + + if ( !is_arch ) return FALSE; + + return TRUE; +} + bool_t bli_cpuid_is_zen ( uint32_t family, @@ -387,6 +427,8 @@ arch_t bli_cpuid_query_id( void ) printf( "features = %u\n", features ); #endif + + if ( vendor == VENDOR_ARM ) { if ( model == MODEL_ARMV8 ) @@ -519,7 +561,7 @@ bool_t bli_cpuid_is_cortexa9 Copyright (C) 2017, The University of Texas at Austin Copyright (C) 2017, Devin Matthews - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_cpuid.h b/frame/base/bli_cpuid.h index b6ecd3d4b..b7ba96ef9 100644 --- a/frame/base/bli_cpuid.h +++ b/frame/base/bli_cpuid.h @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018-2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -60,6 +61,7 @@ bool_t bli_cpuid_is_sandybridge( uint32_t family, uint32_t model, uint32_t fea bool_t bli_cpuid_is_penryn( uint32_t family, uint32_t model, uint32_t features ); // AMD +bool_t bli_cpuid_is_zen2( uint32_t family, uint32_t model, uint32_t features ); bool_t bli_cpuid_is_zen( uint32_t family, uint32_t model, uint32_t features ); bool_t bli_cpuid_is_excavator( uint32_t family, uint32_t model, uint32_t features ); bool_t bli_cpuid_is_steamroller( uint32_t family, uint32_t model, uint32_t features ); diff --git a/frame/base/bli_error.c b/frame/base/bli_error.c index 8ed386af5..54677ae27 100644 --- a/frame/base/bli_error.c +++ b/frame/base/bli_error.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_getopt.c b/frame/base/bli_getopt.c index 222223484..184439db5 100644 --- a/frame/base/bli_getopt.c +++ b/frame/base/bli_getopt.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_gks.c b/frame/base/bli_gks.c index 259075e48..156761f2e 100644 --- a/frame/base/bli_gks.c +++ b/frame/base/bli_gks.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018-2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -97,6 +97,11 @@ void bli_gks_init( void ) #endif // AMD architectures +#ifdef BLIS_CONFIG_ZEN2 + bli_gks_register_cntx( BLIS_ARCH_ZEN2, bli_cntx_init_zen2, + bli_cntx_init_zen2_ref, + bli_cntx_init_zen2_ind ); +#endif #ifdef BLIS_CONFIG_ZEN bli_gks_register_cntx( BLIS_ARCH_ZEN, bli_cntx_init_zen, bli_cntx_init_zen_ref, diff --git a/frame/base/bli_info.c b/frame/base/bli_info.c index 76844ec23..48baffc77 100644 --- a/frame/base/bli_info.c +++ b/frame/base/bli_info.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_info.h b/frame/base/bli_info.h index be078fd7b..8f7869e51 100644 --- a/frame/base/bli_info.h +++ b/frame/base/bli_info.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_init.c b/frame/base/bli_init.c index 1180f1c37..db7e5deec 100644 --- a/frame/base/bli_init.c +++ b/frame/base/bli_init.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_malloc.c b/frame/base/bli_malloc.c index 25ebeb1e0..4e9b10ce3 100644 --- a/frame/base/bli_malloc.c +++ b/frame/base/bli_malloc.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_malloc.h b/frame/base/bli_malloc.h index e7d523a32..2659a81fa 100644 --- a/frame/base/bli_malloc.h +++ b/frame/base/bli_malloc.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_mem.h b/frame/base/bli_mem.h index 5f56f98c0..99977e0c1 100644 --- a/frame/base/bli_mem.h +++ b/frame/base/bli_mem.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_membrk.c b/frame/base/bli_membrk.c index 19b50a52f..762d8c238 100644 --- a/frame/base/bli_membrk.c +++ b/frame/base/bli_membrk.c @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_membrk.h b/frame/base/bli_membrk.h index 4d00eae63..acc026081 100644 --- a/frame/base/bli_membrk.h +++ b/frame/base/bli_membrk.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_memsys.c b/frame/base/bli_memsys.c index 888eb764d..317d3e76d 100644 --- a/frame/base/bli_memsys.c +++ b/frame/base/bli_memsys.c @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_memsys.h b/frame/base/bli_memsys.h index 306819c03..be0d48e35 100644 --- a/frame/base/bli_memsys.h +++ b/frame/base/bli_memsys.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_obj.c b/frame/base/bli_obj.c index f08011e39..f2b59e180 100644 --- a/frame/base/bli_obj.c +++ b/frame/base/bli_obj.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_pool.c b/frame/base/bli_pool.c index 1821e1326..64420655c 100644 --- a/frame/base/bli_pool.c +++ b/frame/base/bli_pool.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_pool.h b/frame/base/bli_pool.h index 0d39fd7d3..69d990edd 100644 --- a/frame/base/bli_pool.h +++ b/frame/base/bli_pool.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_prune.c b/frame/base/bli_prune.c index 080f66f26..ebe5c2365 100644 --- a/frame/base/bli_prune.c +++ b/frame/base/bli_prune.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_rntm.h b/frame/base/bli_rntm.h index 6b686ab4b..c07686414 100644 --- a/frame/base/bli_rntm.h +++ b/frame/base/bli_rntm.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_sba.c b/frame/base/bli_sba.c index 1e2d5753f..2e072504c 100644 --- a/frame/base/bli_sba.c +++ b/frame/base/bli_sba.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_sba.h b/frame/base/bli_sba.h index cf10834e3..f5e36d759 100644 --- a/frame/base/bli_sba.h +++ b/frame/base/bli_sba.h @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/include/bli_arch_config.h b/frame/include/bli_arch_config.h index 000db544c..5aced8886 100644 --- a/frame/include/bli_arch_config.h +++ b/frame/include/bli_arch_config.h @@ -6,6 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP + Copyright (C) 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -61,7 +62,9 @@ CNTX_INIT_PROTS( penryn ) #endif // -- AMD64 architectures -- - +#ifdef BLIS_CONFIG_ZEN2 +CNTX_INIT_PROTS( zen2 ) +#endif #ifdef BLIS_CONFIG_ZEN CNTX_INIT_PROTS( zen ) #endif @@ -153,6 +156,9 @@ CNTX_INIT_PROTS( generic ) // -- AMD64 architectures -- +#ifdef BLIS_FAMILY_ZEN2 +#include "bli_family_zen2.h" +#endif #ifdef BLIS_FAMILY_ZEN #include "bli_family_zen.h" #endif @@ -232,6 +238,9 @@ CNTX_INIT_PROTS( generic ) // -- AMD64 architectures -- +//#ifdef BLIS_KERNELS_ZEN2 +//#include "bli_kernels_zen2.h" +//#endif #ifdef BLIS_KERNELS_ZEN #include "bli_kernels_zen.h" #endif diff --git a/frame/include/bli_macro_defs.h b/frame/include/bli_macro_defs.h index c9f7dacc9..907a5a26c 100644 --- a/frame/include/bli_macro_defs.h +++ b/frame/include/bli_macro_defs.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/include/bli_param_macro_defs.h b/frame/include/bli_param_macro_defs.h index df3ab2135..cc1737e91 100644 --- a/frame/include/bli_param_macro_defs.h +++ b/frame/include/bli_param_macro_defs.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/include/bli_system.h b/frame/include/bli_system.h index c77220862..d91df6803 100644 --- a/frame/include/bli_system.h +++ b/frame/include/bli_system.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/include/bli_type_defs.h b/frame/include/bli_type_defs.h index 5a3bdf513..d3f9c0e1c 100644 --- a/frame/include/bli_type_defs.h +++ b/frame/include/bli_type_defs.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018-2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -992,6 +992,7 @@ typedef enum BLIS_ARCH_PENRYN, // AMD + BLIS_ARCH_ZEN2, BLIS_ARCH_ZEN, BLIS_ARCH_EXCAVATOR, BLIS_ARCH_STEAMROLLER, @@ -1015,7 +1016,9 @@ typedef enum } arch_t; -#define BLIS_NUM_ARCHS 20 +// NOTE: This value must be updated to reflect the number of enum values +// listed above for arch_t! +#define BLIS_NUM_ARCHS 21 // diff --git a/frame/include/blis.h b/frame/include/blis.h index 0deeed1a8..00789c231 100644 --- a/frame/include/blis.h +++ b/frame/include/blis.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/ind/bli_l3_ind.c b/frame/ind/bli_l3_ind.c index a450b7e13..02a7668b8 100644 --- a/frame/ind/bli_l3_ind.c +++ b/frame/ind/bli_l3_ind.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/ind/oapi/bli_l3_3m4m1m_oapi.c b/frame/ind/oapi/bli_l3_3m4m1m_oapi.c index 087e1beef..3d47e6b51 100644 --- a/frame/ind/oapi/bli_l3_3m4m1m_oapi.c +++ b/frame/ind/oapi/bli_l3_3m4m1m_oapi.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/ind/oapi/bli_l3_ind_oapi.c b/frame/ind/oapi/bli_l3_ind_oapi.c index 213753019..85b4b443e 100644 --- a/frame/ind/oapi/bli_l3_ind_oapi.c +++ b/frame/ind/oapi/bli_l3_ind_oapi.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/ind/oapi/bli_l3_nat_oapi.c b/frame/ind/oapi/bli_l3_nat_oapi.c index 52b7e98ad..470db9c8c 100644 --- a/frame/ind/oapi/bli_l3_nat_oapi.c +++ b/frame/ind/oapi/bli_l3_nat_oapi.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_pthread.c b/frame/thread/bli_pthread.c index 03b44a585..20635c123 100644 --- a/frame/thread/bli_pthread.c +++ b/frame/thread/bli_pthread.c @@ -6,7 +6,7 @@ Copyright (C) 2018, Southern Methodist University Copyright (C) 2018, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrcomm.c b/frame/thread/bli_thrcomm.c index c9698050c..39d2204ee 100644 --- a/frame/thread/bli_thrcomm.c +++ b/frame/thread/bli_thrcomm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrcomm.h b/frame/thread/bli_thrcomm.h index 04bceae2a..b4b7a88db 100644 --- a/frame/thread/bli_thrcomm.h +++ b/frame/thread/bli_thrcomm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrcomm_openmp.c b/frame/thread/bli_thrcomm_openmp.c index 05cfa610a..20ab70905 100644 --- a/frame/thread/bli_thrcomm_openmp.c +++ b/frame/thread/bli_thrcomm_openmp.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrcomm_openmp.h b/frame/thread/bli_thrcomm_openmp.h index 4b8956a14..945d9a4b5 100644 --- a/frame/thread/bli_thrcomm_openmp.h +++ b/frame/thread/bli_thrcomm_openmp.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrcomm_pthreads.c b/frame/thread/bli_thrcomm_pthreads.c index 975c5eb88..e4ea4cfa5 100644 --- a/frame/thread/bli_thrcomm_pthreads.c +++ b/frame/thread/bli_thrcomm_pthreads.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrcomm_single.c b/frame/thread/bli_thrcomm_single.c index 969221e7c..d27a114fe 100644 --- a/frame/thread/bli_thrcomm_single.c +++ b/frame/thread/bli_thrcomm_single.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thread.c b/frame/thread/bli_thread.c index 58ba57e81..bbac4b661 100644 --- a/frame/thread/bli_thread.c +++ b/frame/thread/bli_thread.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thread.h b/frame/thread/bli_thread.h index 6680f536e..d3ae01e9c 100644 --- a/frame/thread/bli_thread.h +++ b/frame/thread/bli_thread.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrinfo.c b/frame/thread/bli_thrinfo.c index fdcf31f1d..e37d505cf 100644 --- a/frame/thread/bli_thrinfo.c +++ b/frame/thread/bli_thrinfo.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrinfo.h b/frame/thread/bli_thrinfo.h index 2b3d2e809..6b72c80d6 100644 --- a/frame/thread/bli_thrinfo.h +++ b/frame/thread/bli_thrinfo.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/old/bli_mutex.h b/frame/thread/old/bli_mutex.h index 95d335622..de9f720e8 100644 --- a/frame/thread/old/bli_mutex.h +++ b/frame/thread/old/bli_mutex.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/old/bli_mutex_openmp.h b/frame/thread/old/bli_mutex_openmp.h index f092d7346..9aaa3c79f 100644 --- a/frame/thread/old/bli_mutex_openmp.h +++ b/frame/thread/old/bli_mutex_openmp.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/old/bli_mutex_pthreads.h b/frame/thread/old/bli_mutex_pthreads.h index 7c87dab47..2053e6128 100644 --- a/frame/thread/old/bli_mutex_pthreads.h +++ b/frame/thread/old/bli_mutex_pthreads.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/old/bli_mutex_single.h b/frame/thread/old/bli_mutex_single.h index 0c8db236b..b57d7bba3 100644 --- a/frame/thread/old/bli_mutex_single.h +++ b/frame/thread/old/bli_mutex_single.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/util/bli_util_unb_var1.c b/frame/util/bli_util_unb_var1.c index 8cce17e21..9840e859b 100644 --- a/frame/util/bli_util_unb_var1.c +++ b/frame/util/bli_util_unb_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/haswell/3/bli_gemm_haswell_asm_d6x8.c b/kernels/haswell/3/bli_gemm_haswell_asm_d6x8.c index 48b0394bd..6e16287dc 100644 --- a/kernels/haswell/3/bli_gemm_haswell_asm_d6x8.c +++ b/kernels/haswell/3/bli_gemm_haswell_asm_d6x8.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/haswell/3/bli_gemmtrsm_l_haswell_asm_d6x8.c b/kernels/haswell/3/bli_gemmtrsm_l_haswell_asm_d6x8.c index ae3d67c5f..3d69556ff 100644 --- a/kernels/haswell/3/bli_gemmtrsm_l_haswell_asm_d6x8.c +++ b/kernels/haswell/3/bli_gemmtrsm_l_haswell_asm_d6x8.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/haswell/3/bli_gemmtrsm_u_haswell_asm_d6x8.c b/kernels/haswell/3/bli_gemmtrsm_u_haswell_asm_d6x8.c index ceb4e1e5b..ee54d1e3a 100644 --- a/kernels/haswell/3/bli_gemmtrsm_u_haswell_asm_d6x8.c +++ b/kernels/haswell/3/bli_gemmtrsm_u_haswell_asm_d6x8.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/1/bli_amaxv_zen_int.c b/kernels/zen/1/bli_amaxv_zen_int.c index aa1aa0e66..496649b50 100644 --- a/kernels/zen/1/bli_amaxv_zen_int.c +++ b/kernels/zen/1/bli_amaxv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without @@ -219,6 +219,12 @@ void bli_samaxv_zen_int } } + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // later, especially if BLIS is compiled with -mfpmath=sse). + _mm256_zeroupper(); + /* Store final index to output variable. */ *i_max = i_max_l; } @@ -370,6 +376,12 @@ void bli_damaxv_zen_int } } + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // later, especially if BLIS is compiled with -mfpmath=sse). + _mm256_zeroupper(); + /* Store final index to output variable. */ *i_max = i_max_l; } diff --git a/kernels/zen/1/bli_axpyv_zen_int.c b/kernels/zen/1/bli_axpyv_zen_int.c index 42668a0a7..686580b29 100644 --- a/kernels/zen/1/bli_axpyv_zen_int.c +++ b/kernels/zen/1/bli_axpyv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without @@ -136,6 +136,13 @@ void bli_saxpyv_zen_int y0 += n_elem_per_reg * n_iter_unroll; } + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + const float alphac = *alpha; // If there are leftover iterations, perform them with scalar code. @@ -233,6 +240,13 @@ void bli_daxpyv_zen_int y0 += n_elem_per_reg * n_iter_unroll; } + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + const double alphac = *alpha; // If there are leftover iterations, perform them with scalar code. diff --git a/kernels/zen/1/bli_axpyv_zen_int10.c b/kernels/zen/1/bli_axpyv_zen_int10.c index d2780d39c..873b7da53 100644 --- a/kernels/zen/1/bli_axpyv_zen_int10.c +++ b/kernels/zen/1/bli_axpyv_zen_int10.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without @@ -228,6 +228,13 @@ void bli_saxpyv_zen_int10 y0 += 1*n_elem_per_reg; } + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + for ( ; (i + 0) < n; i += 1 ) { *y0 += (*alpha) * (*x0); @@ -427,6 +434,13 @@ void bli_daxpyv_zen_int10 y0 += 1*n_elem_per_reg; } + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + for ( ; i < n; i += 1 ) { *y0 += (*alpha) * (*x0); diff --git a/kernels/zen/1/bli_dotv_zen_int.c b/kernels/zen/1/bli_dotv_zen_int.c index 1c87a0f87..01022d353 100644 --- a/kernels/zen/1/bli_dotv_zen_int.c +++ b/kernels/zen/1/bli_dotv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without @@ -151,6 +151,13 @@ void bli_sdotv_zen_int rho0 += rho0v.f[0] + rho0v.f[1] + rho0v.f[2] + rho0v.f[3] + rho0v.f[4] + rho0v.f[5] + rho0v.f[6] + rho0v.f[7]; + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + // If there are leftover iterations, perform them with scalar code. for ( i = 0; i < n_left; ++i ) { @@ -265,6 +272,13 @@ void bli_ddotv_zen_int // Accumulate the final rho vector into a single scalar result. rho0 += rho0v.d[0] + rho0v.d[1] + rho0v.d[2] + rho0v.d[3]; + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + // If there are leftover iterations, perform them with scalar code. for ( i = 0; i < n_left; ++i ) { diff --git a/kernels/zen/1/bli_dotv_zen_int10.c b/kernels/zen/1/bli_dotv_zen_int10.c index 79fdde969..e4b980362 100644 --- a/kernels/zen/1/bli_dotv_zen_int10.c +++ b/kernels/zen/1/bli_dotv_zen_int10.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without @@ -224,6 +224,12 @@ void bli_sdotv_zen_int10 // Manually add the results from above to finish the sum. rho0 += rhov[0].f[0] + rhov[0].f[4]; rho0 += rhov[1].f[0] + rhov[1].f[4]; + + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // later, especially if BLIS is compiled with -mfpmath=sse). + _mm256_zeroupper(); } else { @@ -407,6 +413,12 @@ void bli_ddotv_zen_int10 // Manually add the results from above to finish the sum. rho0 += rhov[0].d[0] + rhov[0].d[1] + rhov[0].d[2] + rhov[0].d[3]; rho0 += rhov[1].d[0] + rhov[1].d[1] + rhov[1].d[2] + rhov[1].d[3]; + + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // later, especially if BLIS is compiled with -mfpmath=sse). + _mm256_zeroupper(); } else { diff --git a/kernels/zen/1/bli_dotxv_zen_int.c b/kernels/zen/1/bli_dotxv_zen_int.c index 53b582b77..99ea51710 100644 --- a/kernels/zen/1/bli_dotxv_zen_int.c +++ b/kernels/zen/1/bli_dotxv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without @@ -157,6 +157,13 @@ void bli_sdotxv_zen_int rho0 = rho0v.f[0] + rho0v.f[1] + rho0v.f[2] + rho0v.f[3] + rho0v.f[4] + rho0v.f[5] + rho0v.f[6] + rho0v.f[7]; + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + // If there are leftover iterations, perform them with scalar code. for ( i = 0; i < n_left; ++i ) { @@ -277,6 +284,13 @@ void bli_ddotxv_zen_int // Accumulate the final rho vector into a single scalar result. rho0 = rho0v.d[0] + rho0v.d[1] + rho0v.d[2] + rho0v.d[3]; + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + // If there are leftover iterations, perform them with scalar code. for ( i = 0; i < n_left; ++i ) { diff --git a/kernels/zen/1/bli_scalv_zen_int.c b/kernels/zen/1/bli_scalv_zen_int.c index 3c58212b0..9f76e88e1 100644 --- a/kernels/zen/1/bli_scalv_zen_int.c +++ b/kernels/zen/1/bli_scalv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/1/bli_scalv_zen_int10.c b/kernels/zen/1/bli_scalv_zen_int10.c index 32812d3df..c02af39dd 100644 --- a/kernels/zen/1/bli_scalv_zen_int10.c +++ b/kernels/zen/1/bli_scalv_zen_int10.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/1f/bli_axpyf_zen_int_8.c b/kernels/zen/1f/bli_axpyf_zen_int_8.c index 13bda01e4..3d424e896 100644 --- a/kernels/zen/1f/bli_axpyf_zen_int_8.c +++ b/kernels/zen/1f/bli_axpyf_zen_int_8.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/1f/bli_dotxf_zen_int_8.c b/kernels/zen/1f/bli_dotxf_zen_int_8.c index 4f0734035..ef75eeda1 100644 --- a/kernels/zen/1f/bli_dotxf_zen_int_8.c +++ b/kernels/zen/1f/bli_dotxf_zen_int_8.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/3/bli_gemm_small.c b/kernels/zen/3/bli_gemm_small.c index 776c5a0fe..1db3b59ea 100644 --- a/kernels/zen/3/bli_gemm_small.c +++ b/kernels/zen/3/bli_gemm_small.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -173,28 +173,29 @@ static err_t bli_sgemm_small ) { - int M = bli_obj_length( c ); // number of rows of Matrix C - int N = bli_obj_width( c ); // number of columns of Matrix C - int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + gint_t M = bli_obj_length( c ); // number of rows of Matrix C + gint_t N = bli_obj_width( c ); // number of columns of Matrix C + gint_t K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + gint_t L = M * N; // printf("alpha_cast = %f beta_cast = %f [ Trans = %d %d], [stride = %d %d %d] [m,n,k = %d %d %d]\n",*alpha_cast,*beta_cast, bli_obj_has_trans( a ), bli_obj_has_trans( b ), lda, ldb,ldc, M,N,K); - if (((M * N) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) - || ((M < BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < BLIS_SMALL_K_RECT_MATRIX_THRES))) + if ((((L) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) + || ((M < BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) { - int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. - int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. - int ldc = bli_obj_col_stride( c ); // column stride of matrix C - int row_idx, col_idx, k; + guint_t lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + guint_t ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + guint_t ldc = bli_obj_col_stride( c ); // column stride of matrix C + guint_t row_idx, col_idx, k; float *A = a->buffer; // pointer to elements of Matrix A float *B = b->buffer; // pointer to elements of Matrix B float *C = c->buffer; // pointer to elements of Matrix C float *tA = A, *tB = B, *tC = C;//, *tA_pack; float *tA_packed; // temprorary pointer to hold packed A memory pointer - int row_idx_packed; //packed A memory row index - int lda_packed; //lda of packed A - int col_idx_start; //starting index after A matrix is packed. + guint_t row_idx_packed; //packed A memory row index + guint_t lda_packed; //lda of packed A + guint_t col_idx_start; //starting index after A matrix is packed. dim_t tb_inc_row = 1; // row stride of matrix B dim_t tb_inc_col = ldb; // column stride of matrix B __m256 ymm4, ymm5, ymm6, ymm7; @@ -202,13 +203,13 @@ static err_t bli_sgemm_small __m256 ymm12, ymm13, ymm14, ymm15; __m256 ymm0, ymm1, ymm2, ymm3; - int n_remainder; // If the N is non multiple of 3.(N%3) - int m_remainder; // If the M is non multiple of 32.(M%32) + gint_t n_remainder; // If the N is non multiple of 3.(N%3) + gint_t m_remainder; // If the M is non multiple of 32.(M%32) float *alpha_cast, *beta_cast; // alpha, beta multiples alpha_cast = (alpha->buffer); beta_cast = (beta->buffer); - int required_packing_A = 1; + gint_t required_packing_A = 1; // when N is equal to 1 call GEMV instead of GEMM if (N == 1) @@ -1574,29 +1575,34 @@ static err_t bli_dgemm_small ) { - int M = bli_obj_length( c ); // number of rows of Matrix C - int N = bli_obj_width( c ); // number of columns of Matrix C - int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + gint_t M = bli_obj_length( c ); // number of rows of Matrix C + gint_t N = bli_obj_width( c ); // number of columns of Matrix C + gint_t K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + gint_t L = M * N; // If alpha is zero, scale by beta and return. // printf("alpha_cast = %f beta_cast = %f [ Trans = %d %d], [stride = %d %d %d] [m,n,k = %d %d %d]\n",*alpha_cast,*beta_cast, bli_obj_has_trans( a ), bli_obj_has_trans( b ), lda, ldb,ldc, M,N,K); - if (((M * N) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) - || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if( (L != 0) && (K != 0) && (N < BLIS_SMALL_MATRIX_THRES_ROME) && (K < BLIS_SMALL_MATRIX_THRES_ROME)) +#else + if ((((L) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) + || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) +#endif { - int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. - int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. - int ldc = bli_obj_col_stride( c ); // column stride of matrix C - int row_idx, col_idx, k; + guint_t lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + guint_t ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + guint_t ldc = bli_obj_col_stride( c ); // column stride of matrix C + guint_t row_idx, col_idx, k; double *A = a->buffer; // pointer to elements of Matrix A double *B = b->buffer; // pointer to elements of Matrix B double *C = c->buffer; // pointer to elements of Matrix C double *tA = A, *tB = B, *tC = C;//, *tA_pack; double *tA_packed; // temprorary pointer to hold packed A memory pointer - int row_idx_packed; //packed A memory row index - int lda_packed; //lda of packed A - int col_idx_start; //starting index after A matrix is packed. + guint_t row_idx_packed; //packed A memory row index + guint_t lda_packed; //lda of packed A + guint_t col_idx_start; //starting index after A matrix is packed. dim_t tb_inc_row = 1; // row stride of matrix B dim_t tb_inc_col = ldb; // column stride of matrix B __m256d ymm4, ymm5, ymm6, ymm7; @@ -1604,13 +1610,13 @@ static err_t bli_dgemm_small __m256d ymm12, ymm13, ymm14, ymm15; __m256d ymm0, ymm1, ymm2, ymm3; - int n_remainder; // If the N is non multiple of 3.(N%3) - int m_remainder; // If the M is non multiple of 16.(M%16) + gint_t n_remainder; // If the N is non multiple of 3.(N%3) + gint_t m_remainder; // If the M is non multiple of 16.(M%16) double *alpha_cast, *beta_cast; // alpha, beta multiples alpha_cast = (alpha->buffer); beta_cast = (beta->buffer); - int required_packing_A = 1; + gint_t required_packing_A = 1; // when N is equal to 1 call GEMV instead of GEMM if (N == 1) @@ -2976,12 +2982,12 @@ static err_t bli_sgemm_small_atbn cntl_t* cntl ) { - int M = bli_obj_length( c ); // number of rows of Matrix C - int N = bli_obj_width( c ); // number of columns of Matrix C - int K = bli_obj_length( b ); // number of rows of Matrix B - int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. - int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. - int ldc = bli_obj_col_stride( c ); // column stride of matrix C + gint_t M = bli_obj_length( c ); // number of rows of Matrix C + gint_t N = bli_obj_width( c ); // number of columns of Matrix C + gint_t K = bli_obj_length( b ); // number of rows of Matrix B + guint_t lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + guint_t ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + guint_t ldc = bli_obj_col_stride( c ); // column stride of matrix C int row_idx = 0, col_idx = 0, k; float *A = a->buffer; // pointer to matrix A elements, stored in row major format float *B = b->buffer; // pointer to matrix B elements, stored in column major format @@ -3368,13 +3374,13 @@ static err_t bli_dgemm_small_atbn cntl_t* cntl ) { - int M = bli_obj_length( c ); // number of rows of Matrix C - int N = bli_obj_width( c ); // number of columns of Matrix C - int K = bli_obj_length( b ); // number of rows of Matrix B - int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. - int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. - int ldc = bli_obj_col_stride( c ); // column stride of matrix C - int row_idx = 0, col_idx = 0, k; + gint_t M = bli_obj_length( c ); // number of rows of Matrix C + gint_t N = bli_obj_width( c ); // number of columns of Matrix C + gint_t K = bli_obj_length( b ); // number of rows of Matrix B + guint_t lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + guint_t ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + guint_t ldc = bli_obj_col_stride( c ); // column stride of matrix C + guint_t row_idx = 0, col_idx = 0, k; double *A = a->buffer; // pointer to matrix A elements, stored in row major format double *B = b->buffer; // pointer to matrix B elements, stored in column major format double *C = c->buffer; // pointer to matrix C elements, stored in column major format diff --git a/kernels/zen/3/bli_syrk_small.c b/kernels/zen/3/bli_syrk_small.c new file mode 100644 index 000000000..23d47298c --- /dev/null +++ b/kernels/zen/3/bli_syrk_small.c @@ -0,0 +1,4210 @@ +/* + +BLIS +An object-based framework for developing high-performance BLAS-like +libraries. + +Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of The University of Texas at Austin nor the names +of its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "immintrin.h" +#include "xmmintrin.h" +#include "blis.h" + +#ifdef BLIS_ENABLE_SMALL_MATRIX + +#define MR 32 +#define D_MR (MR >> 1) +#define NR 3 + +#define BLIS_ENABLE_PREFETCH +#define F_SCRATCH_DIM (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES) +static float A_pack[F_SCRATCH_DIM] __attribute__((aligned(64))); +static float C_pack[F_SCRATCH_DIM] __attribute__((aligned(64))); +#define D_BLIS_SMALL_MATRIX_THRES (BLIS_SMALL_MATRIX_THRES / 2 ) +#define D_BLIS_SMALL_M_RECT_MATRIX_THRES (BLIS_SMALL_M_RECT_MATRIX_THRES / 2) +#define D_BLIS_SMALL_K_RECT_MATRIX_THRES (BLIS_SMALL_K_RECT_MATRIX_THRES / 2) +#define D_SCRATCH_DIM (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES) +static double D_A_pack[D_SCRATCH_DIM] __attribute__((aligned(64))); +static double D_C_pack[D_SCRATCH_DIM] __attribute__((aligned(64))); +#define BLIS_ATBN_M_THRES 40 // Threshold value of M for/below which small matrix code is called. +#define AT_MR 4 // The kernel dimension of the A transpose SYRK kernel.(AT_MR * NR). +static err_t bli_ssyrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); + +static err_t bli_dsyrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); + +static err_t bli_ssyrk_small_atbn + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); + +static err_t bli_dsyrk_small_atbn + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); +/* +* The bli_syrk_small function will use the +* custom MRxNR kernels, to perform the computation. +* The custom kernels are used if the [M * N] < 240 * 240 +*/ +err_t bli_syrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + // FGVZ: This code was originally in bli_syrk_front(). However, it really + // fits more naturally here within the bli_syrk_small() function. This + // becomes a bit more obvious now that the code is here, as it contains + // cpp macros such as BLIS_SMALL_MATRIX_A_THRES_M_SYRK, which are specific + // to this implementation. + if ( bli_obj_has_trans( a ) ) + { + // Continue with small implementation. + ; + } + else if ( ( bli_obj_length( a ) <= BLIS_SMALL_MATRIX_A_THRES_M_SYRK && + bli_obj_width( a ) < BLIS_SMALL_MATRIX_A_THRES_N_SYRK ) || + ( bli_obj_length( a ) < BLIS_SMALL_MATRIX_A_THRES_M_SYRK && + bli_obj_width( a ) <= BLIS_SMALL_MATRIX_A_THRES_N_SYRK ) ) + { + // Continue with small implementation. + ; + } + else + { + // Reject the problem and return to large code path. + return BLIS_FAILURE; + } + +#ifdef BLIS_ENABLE_MULTITHREADING + return BLIS_NOT_YET_IMPLEMENTED; +#endif + // If alpha is zero, scale by beta and return. + if (bli_obj_equals(alpha, &BLIS_ZERO)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + // if row major format return. + if ((bli_obj_row_stride( a ) != 1) || + (bli_obj_row_stride( b ) != 1) || + (bli_obj_row_stride( c ) != 1)) + { + return BLIS_INVALID_ROW_STRIDE; + } + + num_t dt = ((*c).info & (0x7 << 0)); + + if (bli_obj_has_trans( a )) + { + if (bli_obj_has_notrans( b )) + { + if (dt == BLIS_FLOAT) + { + return bli_ssyrk_small_atbn(alpha, a, b, beta, c, cntx, cntl); + } + else if (dt == BLIS_DOUBLE) + { + return bli_dsyrk_small_atbn(alpha, a, b, beta, c, cntx, cntl); + } + } + + return BLIS_NOT_YET_IMPLEMENTED; + } + + if (dt == BLIS_DOUBLE) + { + return bli_dsyrk_small(alpha, a, b, beta, c, cntx, cntl); + } + + if (dt == BLIS_FLOAT) + { + return bli_ssyrk_small(alpha, a, b, beta, c, cntx, cntl); + } + + return BLIS_NOT_YET_IMPLEMENTED; +}; + + +static err_t bli_ssyrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + int M = bli_obj_length( c ); // number of rows of Matrix C + int N = bli_obj_width( c ); // number of columns of Matrix C + int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + int L = M * N; + + if ((((L) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) + || ((M < BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) + { + + int lda = bli_obj_col_stride(a); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + int ldb = bli_obj_col_stride(b); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + int ldc_matC = bli_obj_col_stride( c ); // column stride of matrix C + int ldc = M;//bli_obj_col_stride( c ); // column stride of static buffer for matrix C + int row_idx, col_idx, k; + int rs_matC = bli_obj_row_stride( c ); + int rsc = 1; + float *A = a->buffer; // pointer to elements of Matrix A + float *B = b->buffer; // pointer to elements of Matrix B + float *C = C_pack; // pointer to elements of Matrix C + float *matCbuf = c->buffer; + + float *tA = A, *tB = B, *tC = C;//, *tA_pack; + float *tA_packed; // temprorary pointer to hold packed A memory pointer + int row_idx_packed; //packed A memory row index + int lda_packed; //lda of packed A + int col_idx_start; //starting index after A matrix is packed. + dim_t tb_inc_row = 1; // row stride of matrix B + dim_t tb_inc_col = ldb; // column stride of matrix B + __m256 ymm4, ymm5, ymm6, ymm7; + __m256 ymm8, ymm9, ymm10, ymm11; + __m256 ymm12, ymm13, ymm14, ymm15; + __m256 ymm0, ymm1, ymm2, ymm3; + + int n_remainder; // If the N is non multiple of 3.(N%3) + int m_remainder; // If the M is non multiple of 32.(M%32) + + float *alpha_cast, *beta_cast; // alpha, beta multiples + alpha_cast = (alpha->buffer); + beta_cast = (beta->buffer); + int required_packing_A = 1; + + // when N is equal to 1 call GEMV instead of SYRK + if (N == 1) + { + bli_gemv + ( + alpha, + a, + b, + beta, + c + ); + return BLIS_SUCCESS; + } + + //update the pointer math if matrix B needs to be transposed. + if (bli_obj_has_trans( b )) + { + tb_inc_col = 1; //switch row and column strides + tb_inc_row = ldb; + } + + if ((N <= 3) || ((MR * K) > F_SCRATCH_DIM)) + { + required_packing_A = 0; + } + /* + * The computation loop runs for MRxN columns of C matrix, thus + * accessing the MRxK A matrix data and KxNR B matrix data. + * The computation is organized as inner loops of dimension MRxNR. + */ + // Process MR rows of C matrix at a time. + for (row_idx = 0; (row_idx + (MR - 1)) < M; row_idx += MR) + { + + col_idx_start = 0; + tA_packed = A; + row_idx_packed = row_idx; + lda_packed = lda; + + // This is the part of the pack and compute optimization. + // During the first column iteration, we store the accessed A matrix into + // contiguous static memory. This helps to keep te A matrix in Cache and + // aviods the TLB misses. + if (required_packing_A) + { + col_idx = 0; + + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + tA_packed = A_pack; + +#if 0//def BLIS_ENABLE_PREFETCH + _mm_prefetch((char*)(tC + 0), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 16), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc + 16), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc + 16), _MM_HINT_T0); +#endif + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm11 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + // This loop is processing MR x K + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + _mm256_storeu_ps(tA_packed, ymm3); // the packing of matrix A + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + _mm256_storeu_ps(tA_packed + 8, ymm3); // the packing of matrix A + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_ps(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_ps(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + _mm256_storeu_ps(tA_packed + 16, ymm3); // the packing of matrix A + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_ps(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_ps(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_ps(ymm2, ymm3, ymm14); + + ymm3 = _mm256_loadu_ps(tA + 24); + _mm256_storeu_ps(tA_packed + 24, ymm3); // the packing of matrix A + // ymm7 += ymm0 * ymm3; + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + // ymm11 += ymm1 * ymm3; + ymm11 = _mm256_fmadd_ps(ymm1, ymm3, ymm11); + // ymm15 += ymm2 * ymm3; + ymm15 = _mm256_fmadd_ps(ymm2, ymm3, ymm15); + + tA += lda; + tA_packed += MR; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm11 = _mm256_mul_ps(ymm11, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + ymm15 = _mm256_mul_ps(ymm15, ymm0); + + // multiply C by beta and accumulate col 1. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + _mm256_storeu_ps(tC + 16, ymm6); + _mm256_storeu_ps(tC + 24, ymm7); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm11 = _mm256_fmadd_ps(ymm2, ymm1, ymm11);*/ + _mm256_storeu_ps(tC, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + _mm256_storeu_ps(tC + 24, ymm11); + + // multiply C by beta and accumulate, col 3. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm15 = _mm256_fmadd_ps(ymm2, ymm1, ymm15);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + _mm256_storeu_ps(tC + 24, ymm15); + + // modify the pointer arithematic to use packed A matrix. + col_idx_start = NR; + tA_packed = A_pack; + row_idx_packed = 0; + lda_packed = MR; + } + // Process NR columns of C matrix at a time. + for (col_idx = col_idx_start; (col_idx + (NR - 1)) < N; col_idx += NR) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = tA_packed + row_idx_packed; + +#if 0//def BLIS_ENABLE_PREFETCH + _mm_prefetch((char*)(tC + 0), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 16), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc + 16), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc + 16), _MM_HINT_T0); +#endif + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm11 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + // This loop is processing MR x K + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_ps(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_ps(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_ps(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_ps(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_ps(ymm2, ymm3, ymm14); + + ymm3 = _mm256_loadu_ps(tA + 24); + // ymm7 += ymm0 * ymm3; + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + // ymm11 += ymm1 * ymm3; + ymm11 = _mm256_fmadd_ps(ymm1, ymm3, ymm11); + // ymm15 += ymm2 * ymm3; + ymm15 = _mm256_fmadd_ps(ymm2, ymm3, ymm15); + + tA += lda_packed; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm11 = _mm256_mul_ps(ymm11, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + ymm15 = _mm256_mul_ps(ymm15, ymm0); + + // multiply C by beta and accumulate col 1. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + _mm256_storeu_ps(tC + 16, ymm6); + _mm256_storeu_ps(tC + 24, ymm7); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm11 = _mm256_fmadd_ps(ymm2, ymm1, ymm11);*/ + _mm256_storeu_ps(tC, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + _mm256_storeu_ps(tC + 24, ymm11); + + // multiply C by beta and accumulate, col 3. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm15 = _mm256_fmadd_ps(ymm2, ymm1, ymm15);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + _mm256_storeu_ps(tC + 24, ymm15); + + } + n_remainder = N - col_idx; + + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm11 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm8 = _mm256_fmadd_ps(ymm0, ymm3, ymm8); + ymm12 = _mm256_fmadd_ps(ymm1, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm9 = _mm256_fmadd_ps(ymm0, ymm3, ymm9); + ymm13 = _mm256_fmadd_ps(ymm1, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + ymm14 = _mm256_fmadd_ps(ymm1, ymm3, ymm14); + + ymm3 = _mm256_loadu_ps(tA + 24); + ymm11 = _mm256_fmadd_ps(ymm0, ymm3, ymm11); + ymm15 = _mm256_fmadd_ps(ymm1, ymm3, ymm15); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm11 = _mm256_mul_ps(ymm11, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + ymm15 = _mm256_mul_ps(ymm15, ymm0); + + // multiply C by beta and accumulate, col 1. + /*ymm2 = _mm256_loadu_ps(tC + 0); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm11 = _mm256_fmadd_ps(ymm2, ymm1, ymm11);*/ + _mm256_storeu_ps(tC + 0, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + _mm256_storeu_ps(tC + 24, ymm11); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm15 = _mm256_fmadd_ps(ymm2, ymm1, ymm15);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + _mm256_storeu_ps(tC + 24, ymm15); + + col_idx += 2; + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm12 = _mm256_fmadd_ps(ymm0, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + ymm14 = _mm256_fmadd_ps(ymm0, ymm3, ymm14); + + ymm3 = _mm256_loadu_ps(tA + 24); + ymm15 = _mm256_fmadd_ps(ymm0, ymm3, ymm15); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + ymm15 = _mm256_mul_ps(ymm15, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC + 0); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm15 = _mm256_fmadd_ps(ymm2, ymm1, ymm15);*/ + + _mm256_storeu_ps(tC + 0, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + _mm256_storeu_ps(tC + 24, ymm15); + } + } + + m_remainder = M - row_idx; + + if (m_remainder >= 24) + { + m_remainder -= 24; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_ps(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_ps(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_ps(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_ps(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_ps(ymm2, ymm3, ymm14); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + _mm256_storeu_ps(tC + 16, ymm6); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10);*/ + _mm256_storeu_ps(tC, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm8 = _mm256_fmadd_ps(ymm0, ymm3, ymm8); + ymm12 = _mm256_fmadd_ps(ymm1, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm9 = _mm256_fmadd_ps(ymm0, ymm3, ymm9); + ymm13 = _mm256_fmadd_ps(ymm1, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + ymm14 = _mm256_fmadd_ps(ymm1, ymm3, ymm14); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC + 0); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10);*/ + _mm256_storeu_ps(tC + 0, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + + col_idx += 2; + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm12 = _mm256_fmadd_ps(ymm0, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + ymm14 = _mm256_fmadd_ps(ymm0, ymm3, ymm14); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC + 0); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14);*/ + + _mm256_storeu_ps(tC + 0, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + } + + row_idx += 24; + } + + if (m_remainder >= 16) + { + m_remainder -= 16; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm6 = _mm256_fmadd_ps(ymm1, ymm3, ymm6); + ymm8 = _mm256_fmadd_ps(ymm2, ymm3, ymm8); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(tC, ymm6); + _mm256_storeu_ps(tC + 8, ymm7); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9);*/ + _mm256_storeu_ps(tC, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm6 = _mm256_fmadd_ps(ymm1, ymm3, ymm6); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(tC, ymm6); + _mm256_storeu_ps(tC + 8, ymm7); + + col_idx += 2; + + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + + } + + row_idx += 16; + } + + if (m_remainder >= 8) + { + m_remainder -= 8; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_ps(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_ps(ymm2, ymm3, ymm6); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4);*/ + _mm256_storeu_ps(tC, ymm4); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6);*/ + _mm256_storeu_ps(tC, ymm6); + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_ps(ymm1, ymm3, ymm5); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4);*/ + _mm256_storeu_ps(tC, ymm4); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm5); + + col_idx += 2; + + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + ymm4 = _mm256_mul_ps(ymm4, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4);*/ + _mm256_storeu_ps(tC, ymm4); + + } + + row_idx += 8; + } + // M is not a multiple of 32. + // The handling of edge case where the remainder + // dimension is less than 8. The padding takes place + // to handle this case. + if ((m_remainder) && (lda > 7)) + { + float f_temp[8]; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm5 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(f_temp, ymm7); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9);*/ + _mm256_storeu_ps(f_temp, ymm9); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm5 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + ymm3 = _mm256_loadu_ps(tA); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + + tA += lda; + } + + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(f_temp, ymm7); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm5 = _mm256_setzero_ps(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + ymm3 = _mm256_loadu_ps(tA); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + + tA += lda; + } + + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + // multiply C by beta and accumulate. + ymm5 = _mm256_mul_ps(ymm5, ymm0); + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + m_remainder = 0; + } + + if (m_remainder) + { + float result; + for (; row_idx < M; row_idx += 1) + { + for (col_idx = 0; col_idx < N; col_idx += 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + result = 0; + for (k = 0; k < K; ++k) + { + result += (*tA) * (*tB); + tA += lda; + tB += tb_inc_row; + } + + result *= (*alpha_cast); + (*tC) = /*(*tC) * (*beta_cast) + */result; + } + } + } + + //copy/compute sryk values back to C using SIMD + if ( bli_seq0( *beta_cast ) ) + {//just copy in case of beta = 0 + dim_t _i, _j, k, _l; + if(bli_obj_is_lower(c)) // c is lower + { + //first column + _j = 0; + k = M >> 3; + _i = 0; + for ( _l = 0; _l < k; _l++ ) + { + ymm0 = _mm256_loadu_ps((C + _i*rsc)); + _mm256_storeu_ps((matCbuf + _i*rs_matC), ymm0); + _i += 8; + } + while (_i < M ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + while ( _j < N ) //next column + { + //k = (_j + (8 - (_j & 7))); + _l = _j & 7; + k = (_l != 0) ? (_j + (8 - _l)) : _j; + k = (k <= M) ? k : M; + for ( _i = _j; _i < k; ++_i ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + k = (M - _i) >> 3; + _l = 0; + while ( _l < k ) + { + ymm0 = _mm256_loadu_ps((C + _i*rsc + _j*ldc)); + _mm256_storeu_ps((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + + _i += 8; + _l++; + } + while (_i < M ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + { + k = (_j + 1) >> 3; + _i = 0; + _l = 0; + while ( _l < k ) + { + ymm0 = _mm256_loadu_ps((C + _i*rsc + _j*ldc)); + _mm256_storeu_ps((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + _i += 8; + _l++; + } + while (_i <= _j ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + ++_i; + } + } + } + } + else + {//when beta is non-zero, fmadd and store the results + dim_t _i, _j, k, _l; + ymm1 = _mm256_broadcast_ss(beta_cast); + if(bli_obj_is_lower(c)) //c is lower + { + //first column + _j = 0; + k = M >> 3; + _i = 0; + for ( _l = 0; _l < k; _l++ ) + { + ymm2 = _mm256_loadu_ps((matCbuf + _i*rs_matC)); + ymm0 = _mm256_loadu_ps((C + _i*rsc)); + ymm0 = _mm256_fmadd_ps(ymm2, ymm1, ymm0); + _mm256_storeu_ps((matCbuf + _i*rs_matC), ymm0); + _i += 8; + } + while (_i < M ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + while ( _j < N ) //next column + { + //k = (_j + (8 - (_j & 7))); + _l = _j & 7; + k = (_l != 0) ? (_j + (8 - _l)) : _j; + k = (k <= M) ? k : M; + for ( _i = _j; _i < k; ++_i ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + k = (M - _i) >> 3; + _l = 0; + while ( _l < k ) + { + ymm2 = _mm256_loadu_ps((matCbuf + _i*rs_matC + _j*ldc_matC)); + ymm0 = _mm256_loadu_ps((C + _i*rsc + _j*ldc)); + ymm0 = _mm256_fmadd_ps(ymm2, ymm1, ymm0); + _mm256_storeu_ps((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + + _i += 8; + _l++; + } + while (_i < M ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + { + k = (_j + 1) >> 3; + _i = 0; + _l = 0; + while ( _l < k ) + { + ymm2 = _mm256_loadu_ps((matCbuf + _i*rs_matC + _j*ldc_matC)); + ymm0 = _mm256_loadu_ps((C + _i*rsc + _j*ldc)); + ymm0 = _mm256_fmadd_ps(ymm2, ymm1, ymm0); + _mm256_storeu_ps((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + _i += 8; + _l++; + } + while (_i <= _j ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + ++_i; + } + } + } + } + + return BLIS_SUCCESS; + } + else + return BLIS_NONCONFORMAL_DIMENSIONS; + + +}; + +static err_t bli_dsyrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + int M = bli_obj_length( c ); // number of rows of Matrix C + int N = bli_obj_width( c ); // number of columns of Matrix C + int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + int L = M * N; + + // If alpha is zero, scale by beta and return. + if ((((L) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) + || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) + { + + int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + int ldc_matC = bli_obj_col_stride( c ); // column stride of matrix C + int ldc = M;//bli_obj_col_stride( c ); // column stride of static buffer for matrix C + int row_idx, col_idx, k; + int rs_matC = bli_obj_row_stride( c ); + int rsc = 1; + double *A = a->buffer; // pointer to elements of Matrix A + double *B = b->buffer; // pointer to elements of Matrix B + double *C = D_C_pack; // pointer to elements of Matrix C + double *matCbuf = c->buffer; + + double *tA = A, *tB = B, *tC = C;//, *tA_pack; + double *tA_packed; // temprorary pointer to hold packed A memory pointer + int row_idx_packed; //packed A memory row index + int lda_packed; //lda of packed A + int col_idx_start; //starting index after A matrix is packed. + dim_t tb_inc_row = 1; // row stride of matrix B + dim_t tb_inc_col = ldb; // column stride of matrix B + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm0, ymm1, ymm2, ymm3; + + int n_remainder; // If the N is non multiple of 3.(N%3) + int m_remainder; // If the M is non multiple of 16.(M%16) + + double *alpha_cast, *beta_cast; // alpha, beta multiples + alpha_cast = (alpha->buffer); + beta_cast = (beta->buffer); + int required_packing_A = 1; + + // when N is equal to 1 call GEMV instead of SYRK + if (N == 1) + { + bli_gemv + ( + alpha, + a, + b, + beta, + c + ); + return BLIS_SUCCESS; + } + + //update the pointer math if matrix B needs to be transposed. + if (bli_obj_has_trans( b )) + { + tb_inc_col = 1; //switch row and column strides + tb_inc_row = ldb; + } + + if ((N <= 3) || ((D_MR * K) > D_SCRATCH_DIM)) + { + required_packing_A = 0; + } + /* + * The computation loop runs for D_MRxN columns of C matrix, thus + * accessing the D_MRxK A matrix data and KxNR B matrix data. + * The computation is organized as inner loops of dimension D_MRxNR. + */ + // Process D_MR rows of C matrix at a time. + for (row_idx = 0; (row_idx + (D_MR - 1)) < M; row_idx += D_MR) + { + + col_idx_start = 0; + tA_packed = A; + row_idx_packed = row_idx; + lda_packed = lda; + + // This is the part of the pack and compute optimization. + // During the first column iteration, we store the accessed A matrix into + // contiguous static memory. This helps to keep te A matrix in Cache and + // aviods the TLB misses. + if (required_packing_A) + { + col_idx = 0; + + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + tA_packed = D_A_pack; + +#if 0//def BLIS_ENABLE_PREFETCH + _mm_prefetch((char*)(tC + 0), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 8), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc + 8), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc + 8), _MM_HINT_T0); +#endif + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + // This loop is processing D_MR x K + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + _mm256_storeu_pd(tA_packed, ymm3); // the packing of matrix A + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + _mm256_storeu_pd(tA_packed + 4, ymm3); // the packing of matrix A + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_pd(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_pd(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + _mm256_storeu_pd(tA_packed + 8, ymm3); // the packing of matrix A + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_pd(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_pd(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_pd(ymm2, ymm3, ymm14); + + ymm3 = _mm256_loadu_pd(tA + 12); + _mm256_storeu_pd(tA_packed + 12, ymm3); // the packing of matrix A + // ymm7 += ymm0 * ymm3; + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + // ymm11 += ymm1 * ymm3; + ymm11 = _mm256_fmadd_pd(ymm1, ymm3, ymm11); + // ymm15 += ymm2 * ymm3; + ymm15 = _mm256_fmadd_pd(ymm2, ymm3, ymm15); + + tA += lda; + tA_packed += D_MR; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm11 = _mm256_mul_pd(ymm11, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + // multiply C by beta and accumulate col 1. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + _mm256_storeu_pd(tC + 8, ymm6); + _mm256_storeu_pd(tC + 12, ymm7); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm11 = _mm256_fmadd_pd(ymm2, ymm1, ymm11);*/ + _mm256_storeu_pd(tC, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + _mm256_storeu_pd(tC + 12, ymm11); + + // multiply C by beta and accumulate, col 3. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm15 = _mm256_fmadd_pd(ymm2, ymm1, ymm15);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + _mm256_storeu_pd(tC + 12, ymm15); + + // modify the pointer arithematic to use packed A matrix. + col_idx_start = NR; + tA_packed = D_A_pack; + row_idx_packed = 0; + lda_packed = D_MR; + } + // Process NR columns of C matrix at a time. + for (col_idx = col_idx_start; (col_idx + (NR - 1)) < N; col_idx += NR) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = tA_packed + row_idx_packed; + +#if 0//def BLIS_ENABLE_PREFETCH + _mm_prefetch((char*)(tC + 0), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 8), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc + 8), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc + 8), _MM_HINT_T0); +#endif + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + // This loop is processing D_MR x K + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_pd(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_pd(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_pd(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_pd(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_pd(ymm2, ymm3, ymm14); + + ymm3 = _mm256_loadu_pd(tA + 12); + // ymm7 += ymm0 * ymm3; + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + // ymm11 += ymm1 * ymm3; + ymm11 = _mm256_fmadd_pd(ymm1, ymm3, ymm11); + // ymm15 += ymm2 * ymm3; + ymm15 = _mm256_fmadd_pd(ymm2, ymm3, ymm15); + + tA += lda_packed; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm11 = _mm256_mul_pd(ymm11, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + // multiply C by beta and accumulate col 1. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + _mm256_storeu_pd(tC + 8, ymm6); + _mm256_storeu_pd(tC + 12, ymm7); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm11 = _mm256_fmadd_pd(ymm2, ymm1, ymm11);*/ + _mm256_storeu_pd(tC, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + _mm256_storeu_pd(tC + 12, ymm11); + + // multiply C by beta and accumulate, col 3. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm15 = _mm256_fmadd_pd(ymm2, ymm1, ymm15);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + _mm256_storeu_pd(tC + 12, ymm15); + + } + n_remainder = N - col_idx; + + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm8 = _mm256_fmadd_pd(ymm0, ymm3, ymm8); + ymm12 = _mm256_fmadd_pd(ymm1, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm9 = _mm256_fmadd_pd(ymm0, ymm3, ymm9); + ymm13 = _mm256_fmadd_pd(ymm1, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + ymm14 = _mm256_fmadd_pd(ymm1, ymm3, ymm14); + + ymm3 = _mm256_loadu_pd(tA + 12); + ymm11 = _mm256_fmadd_pd(ymm0, ymm3, ymm11); + ymm15 = _mm256_fmadd_pd(ymm1, ymm3, ymm15); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm11 = _mm256_mul_pd(ymm11, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + // multiply C by beta and accumulate, col 1. + /*ymm2 = _mm256_loadu_pd(tC + 0); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm11 = _mm256_fmadd_pd(ymm2, ymm1, ymm11);*/ + _mm256_storeu_pd(tC + 0, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + _mm256_storeu_pd(tC + 12, ymm11); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm15 = _mm256_fmadd_pd(ymm2, ymm1, ymm15);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + _mm256_storeu_pd(tC + 12, ymm15); + + col_idx += 2; + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm12 = _mm256_fmadd_pd(ymm0, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + ymm14 = _mm256_fmadd_pd(ymm0, ymm3, ymm14); + + ymm3 = _mm256_loadu_pd(tA + 12); + ymm15 = _mm256_fmadd_pd(ymm0, ymm3, ymm15); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC + 0); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm15 = _mm256_fmadd_pd(ymm2, ymm1, ymm15);*/ + + _mm256_storeu_pd(tC + 0, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + _mm256_storeu_pd(tC + 12, ymm15); + } + } + + m_remainder = M - row_idx; + + if (m_remainder >= 12) + { + m_remainder -= 12; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_pd(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_pd(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_pd(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_pd(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_pd(ymm2, ymm3, ymm14); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + _mm256_storeu_pd(tC + 8, ymm6); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10);*/ + _mm256_storeu_pd(tC, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm8 = _mm256_fmadd_pd(ymm0, ymm3, ymm8); + ymm12 = _mm256_fmadd_pd(ymm1, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm9 = _mm256_fmadd_pd(ymm0, ymm3, ymm9); + ymm13 = _mm256_fmadd_pd(ymm1, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + ymm14 = _mm256_fmadd_pd(ymm1, ymm3, ymm14); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC + 0); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10);*/ + _mm256_storeu_pd(tC + 0, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + + col_idx += 2; + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm12 = _mm256_fmadd_pd(ymm0, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + ymm14 = _mm256_fmadd_pd(ymm0, ymm3, ymm14); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC + 0); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14);*/ + + _mm256_storeu_pd(tC + 0, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + } + + row_idx += 12; + } + + if (m_remainder >= 8) + { + m_remainder -= 8; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm6 = _mm256_fmadd_pd(ymm1, ymm3, ymm6); + ymm8 = _mm256_fmadd_pd(ymm2, ymm3, ymm8); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(tC, ymm6); + _mm256_storeu_pd(tC + 4, ymm7); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9);*/ + _mm256_storeu_pd(tC, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm6 = _mm256_fmadd_pd(ymm1, ymm3, ymm6); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(tC, ymm6); + _mm256_storeu_pd(tC + 4, ymm7); + + col_idx += 2; + + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + + } + + row_idx += 8; + } + + if (m_remainder >= 4) + { + m_remainder -= 4; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_pd(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_pd(ymm2, ymm3, ymm6); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4);*/ + _mm256_storeu_pd(tC, ymm4); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6);*/ + _mm256_storeu_pd(tC, ymm6); + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_pd(ymm1, ymm3, ymm5); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4);*/ + _mm256_storeu_pd(tC, ymm4); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm5); + + col_idx += 2; + + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + ymm4 = _mm256_mul_pd(ymm4, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4);*/ + _mm256_storeu_pd(tC, ymm4); + + } + + row_idx += 4; + } + // M is not a multiple of 32. + // The handling of edge case where the remainder + // dimension is less than 8. The padding takes place + // to handle this case. + if ((m_remainder) && (lda > 3)) + { + double f_temp[8]; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm5 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(f_temp, ymm7); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9);*/ + _mm256_storeu_pd(f_temp, ymm9); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm5 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + ymm3 = _mm256_loadu_pd(tA); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + + tA += lda; + } + + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(f_temp, ymm7); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm5 = _mm256_setzero_pd(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + ymm3 = _mm256_loadu_pd(tA); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + + tA += lda; + } + + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + // multiply C by beta and accumulate. + ymm5 = _mm256_mul_pd(ymm5, ymm0); + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + m_remainder = 0; + } + + if (m_remainder) + { + double result; + for (; row_idx < M; row_idx += 1) + { + for (col_idx = 0; col_idx < N; col_idx += 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + result = 0; + for (k = 0; k < K; ++k) + { + result += (*tA) * (*tB); + tA += lda; + tB += tb_inc_row; + } + + result *= (*alpha_cast); + (*tC) = /*(*tC) * (*beta_cast) + */result; + } + } + } + + //copy/compute sryk values back to C using SIMD + if ( bli_seq0( *beta_cast ) ) + {//just copy for beta = 0 + dim_t _i, _j, k, _l; + if(bli_obj_is_lower(c)) //c is lower + { + //first column + _j = 0; + k = M >> 2; + _i = 0; + for ( _l = 0; _l < k; _l++ ) + { + ymm0 = _mm256_loadu_pd((C + _i*rsc)); + _mm256_storeu_pd((matCbuf + _i*rs_matC), ymm0); + _i += 4; + } + while (_i < M ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + while ( _j < N ) //next column + { + //k = (_j + (4 - (_j & 3))); + _l = _j & 3; + k = (_l != 0) ? (_j + (4 - _l)) : _j; + k = (k <= M) ? k : M; + for ( _i = _j; _i < k; ++_i ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + k = (M - _i) >> 2; + _l = 0; + while ( _l < k ) + { + ymm0 = _mm256_loadu_pd((C + _i*rsc + _j*ldc)); + _mm256_storeu_pd((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + + _i += 4; + _l++; + } + while (_i < M ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + { + k = (_j + 1) >> 2; + _i = 0; + _l = 0; + while ( _l < k ) + { + ymm0 = _mm256_loadu_pd((C + _i*rsc + _j*ldc)); + _mm256_storeu_pd((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + _i += 4; + _l++; + } + while (_i <= _j ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + ++_i; + } + } + } + } + else + {//when beta is non-zero, fmadd and store the results + dim_t _i, _j, k, _l; + ymm1 = _mm256_broadcast_sd(beta_cast); + if(bli_obj_is_lower(c)) //c is lower + { + //first column + _j = 0; + k = M >> 2; + _i = 0; + for ( _l = 0; _l < k; _l++ ) + { + ymm2 = _mm256_loadu_pd((matCbuf + _i*rs_matC)); + ymm0 = _mm256_loadu_pd((C + _i*rsc)); + ymm0 = _mm256_fmadd_pd(ymm2, ymm1, ymm0); + _mm256_storeu_pd((matCbuf + _i*rs_matC), ymm0); + _i += 4; + } + while (_i < M ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + while ( _j < N ) //next column + { + //k = (_j + (4 - (_j & 3))); + _l = _j & 3; + k = (_l != 0) ? (_j + (4 - _l)) : _j; + k = (k <= M) ? k : M; + for ( _i = _j; _i < k; ++_i ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + k = (M - _i) >> 2; + _l = 0; + while ( _l < k ) + { + ymm2 = _mm256_loadu_pd((matCbuf + _i*rs_matC + _j*ldc_matC)); + ymm0 = _mm256_loadu_pd((C + _i*rsc + _j*ldc)); + ymm0 = _mm256_fmadd_pd(ymm2, ymm1, ymm0); + _mm256_storeu_pd((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + + _i += 4; + _l++; + } + while (_i < M ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + { + k = (_j + 1) >> 2; + _i = 0; + _l = 0; + while ( _l < k ) + { + ymm2 = _mm256_loadu_pd((matCbuf + _i*rs_matC + _j*ldc_matC)); + ymm0 = _mm256_loadu_pd((C + _i*rsc + _j*ldc)); + ymm0 = _mm256_fmadd_pd(ymm2, ymm1, ymm0); + _mm256_storeu_pd((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + _i += 4; + _l++; + } + while (_i <= _j ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + ++_i; + } + } + } + } + + return BLIS_SUCCESS; + } + else + return BLIS_NONCONFORMAL_DIMENSIONS; + + +}; + +static err_t bli_ssyrk_small_atbn + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int M = bli_obj_length(c); // number of rows of Matrix C + int N = bli_obj_width(c); // number of columns of Matrix C + int K = bli_obj_length(b); // number of rows of Matrix B + int lda = bli_obj_col_stride(a); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + int ldb = bli_obj_col_stride(b); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + int ldc_matC = bli_obj_col_stride( c ); // column stride of matrix C + int ldc = M;//bli_obj_col_stride( c ); // column stride of static buffer for matrix C + int row_idx = 0, col_idx = 0, k; + int rs_matC = bli_obj_row_stride( c ); + int rsc = 1; + float *A = a->buffer; // pointer to matrix A elements, stored in row major format + float *B = b->buffer; // pointer to matrix B elements, stored in column major format + float *C = C_pack; // pointer to matrix C elements, stored in column major format + float *matCbuf = c->buffer; + + float *tA = A, *tB = B, *tC = C; + + __m256 ymm4, ymm5, ymm6, ymm7; + __m256 ymm8, ymm9, ymm10, ymm11; + __m256 ymm12, ymm13, ymm14, ymm15; + __m256 ymm0, ymm1, ymm2, ymm3; + + float result, scratch[8]; + float *alpha_cast, *beta_cast; // alpha, beta multiples + alpha_cast = (alpha->buffer); + beta_cast = (beta->buffer); + + // The non-copy version of the A^T SYRK gives better performance for the small M cases. + // The threshold is controlled by BLIS_ATBN_M_THRES + if (M <= BLIS_ATBN_M_THRES) + { + for (col_idx = 0; (col_idx + (NR - 1)) < N; col_idx += NR) + { + for (row_idx = 0; (row_idx + (AT_MR - 1)) < M; row_idx += AT_MR) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm11 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + //The inner loop computes the 4x3 values of the matrix. + //The computation pattern is: + // ymm4 ymm5 ymm6 + // ymm7 ymm8 ymm9 + // ymm10 ymm11 ymm12 + // ymm13 ymm14 ymm15 + + //The Dot operation is performed in the inner loop, 8 float elements fit + //in the YMM register hence loop count incremented by 8 + for (k = 0; (k + 7) < K; k += 8) + { + ymm0 = _mm256_loadu_ps(tB + 0); + ymm1 = _mm256_loadu_ps(tB + ldb); + ymm2 = _mm256_loadu_ps(tB + 2 * ldb); + + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_ps(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_ps(ymm2, ymm3, ymm6); + + ymm3 = _mm256_loadu_ps(tA + lda); + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + ymm3 = _mm256_loadu_ps(tA + 2 * lda); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + ymm11 = _mm256_fmadd_ps(ymm1, ymm3, ymm11); + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 3 * lda); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + ymm14 = _mm256_fmadd_ps(ymm1, ymm3, ymm14); + ymm15 = _mm256_fmadd_ps(ymm2, ymm3, ymm15); + + tA += 8; + tB += 8; + + } + + // if K is not a multiple of 8, padding is done before load using temproary array. + if (k < K) + { + int iter; + float data_feeder[8] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_ps(data_feeder); + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter + ldb]; + ymm1 = _mm256_loadu_ps(data_feeder); + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter + 2 * ldb]; + ymm2 = _mm256_loadu_ps(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_ps(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_ps(ymm2, ymm3, ymm6); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[2 * lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + ymm11 = _mm256_fmadd_ps(ymm1, ymm3, ymm11); + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[3 * lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + ymm14 = _mm256_fmadd_ps(ymm1, ymm3, ymm14); + ymm15 = _mm256_fmadd_ps(ymm2, ymm3, ymm15); + + } + + //horizontal addition and storage of the data. + //Results for 4x3 blocks of C is stored here + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + _mm256_storeu_ps(scratch, ymm4); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm7 = _mm256_hadd_ps(ymm7, ymm7); + ymm7 = _mm256_hadd_ps(ymm7, ymm7); + _mm256_storeu_ps(scratch, ymm7); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm10 = _mm256_hadd_ps(ymm10, ymm10); + ymm10 = _mm256_hadd_ps(ymm10, ymm10); + _mm256_storeu_ps(scratch, ymm10); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm13 = _mm256_hadd_ps(ymm13, ymm13); + ymm13 = _mm256_hadd_ps(ymm13, ymm13); + _mm256_storeu_ps(scratch, ymm13); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + tC += ldc; + ymm5 = _mm256_hadd_ps(ymm5, ymm5); + ymm5 = _mm256_hadd_ps(ymm5, ymm5); + _mm256_storeu_ps(scratch, ymm5); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm8 = _mm256_hadd_ps(ymm8, ymm8); + ymm8 = _mm256_hadd_ps(ymm8, ymm8); + _mm256_storeu_ps(scratch, ymm8); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm11 = _mm256_hadd_ps(ymm11, ymm11); + ymm11 = _mm256_hadd_ps(ymm11, ymm11); + _mm256_storeu_ps(scratch, ymm11); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm14 = _mm256_hadd_ps(ymm14, ymm14); + ymm14 = _mm256_hadd_ps(ymm14, ymm14); + _mm256_storeu_ps(scratch, ymm14); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + tC += ldc; + ymm6 = _mm256_hadd_ps(ymm6, ymm6); + ymm6 = _mm256_hadd_ps(ymm6, ymm6); + _mm256_storeu_ps(scratch, ymm6); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm9 = _mm256_hadd_ps(ymm9, ymm9); + ymm9 = _mm256_hadd_ps(ymm9, ymm9); + _mm256_storeu_ps(scratch, ymm9); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm12 = _mm256_hadd_ps(ymm12, ymm12); + ymm12 = _mm256_hadd_ps(ymm12, ymm12); + _mm256_storeu_ps(scratch, ymm12); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm15 = _mm256_hadd_ps(ymm15, ymm15); + ymm15 = _mm256_hadd_ps(ymm15, ymm15); + _mm256_storeu_ps(scratch, ymm15); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + } + } + + int processed_col = col_idx; + int processed_row = row_idx; + + //The edge case handling where N is not a multiple of 3 + if (processed_col < N) + { + for (col_idx = processed_col; col_idx < N; col_idx += 1) + { + for (row_idx = 0; (row_idx + (AT_MR - 1)) < M; row_idx += AT_MR) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + + //The inner loop computes the 4x1 values of the matrix. + //The computation pattern is: + // ymm4 + // ymm7 + // ymm10 + // ymm13 + + for (k = 0; (k + 7) < K; k += 8) + { + ymm0 = _mm256_loadu_ps(tB + 0); + + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + ymm3 = _mm256_loadu_ps(tA + lda); + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + + ymm3 = _mm256_loadu_ps(tA + 2 * lda); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + + ymm3 = _mm256_loadu_ps(tA + 3 * lda); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + + tA += 8; + tB += 8; + } + + // if K is not a multiple of 8, padding is done before load using temproary array. + if (k < K) + { + int iter; + float data_feeder[8] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_ps(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[2 * lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[3 * lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + + } + + //horizontal addition and storage of the data. + //Results for 4x1 blocks of C is stored here + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + _mm256_storeu_ps(scratch, ymm4); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm7 = _mm256_hadd_ps(ymm7, ymm7); + ymm7 = _mm256_hadd_ps(ymm7, ymm7); + _mm256_storeu_ps(scratch, ymm7); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm10 = _mm256_hadd_ps(ymm10, ymm10); + ymm10 = _mm256_hadd_ps(ymm10, ymm10); + _mm256_storeu_ps(scratch, ymm10); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm13 = _mm256_hadd_ps(ymm13, ymm13); + ymm13 = _mm256_hadd_ps(ymm13, ymm13); + _mm256_storeu_ps(scratch, ymm13); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + } + } + processed_row = row_idx; + } + + //The edge case handling where M is not a multiple of 4 + if (processed_row < M) + { + for (row_idx = processed_row; row_idx < M; row_idx += 1) + { + for (col_idx = 0; col_idx < N; col_idx += 1) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + + for (k = 0; (k + 7) < K; k += 8) + { + ymm0 = _mm256_loadu_ps(tB + 0); + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + tA += 8; + tB += 8; + } + + // if K is not a multiple of 8, padding is done before load using temproary array. + if (k < K) + { + int iter; + float data_feeder[8] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_ps(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + } + + //horizontal addition and storage of the data. + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + _mm256_storeu_ps(scratch, ymm4); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + } + } + } + + //copy/compute sryk values back to C + if ( bli_seq0( *beta_cast ) ) //when beta is 0, just copy result to C + { + dim_t _i, _j; + if(bli_obj_is_lower(c)) //c is lower + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i <= 0 ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i >= 0 ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + } + else //when beta is non-zero, multiply and store result to C + { + dim_t _i, _j; + if(bli_obj_is_lower(c)) //c is lower + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i <= 0 ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i >= 0 ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + } + + return BLIS_SUCCESS; + } + else + return BLIS_NONCONFORMAL_DIMENSIONS; +} + +static err_t bli_dsyrk_small_atbn + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int M = bli_obj_length( c ); // number of rows of Matrix C + int N = bli_obj_width( c ); // number of columns of Matrix C + int K = bli_obj_length( b ); // number of rows of Matrix B + int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + int ldc_matC = bli_obj_col_stride( c ); // column stride of matrix C + int ldc = M;//bli_obj_col_stride( c ); // column stride of static buffer for matrix C + int row_idx = 0, col_idx = 0, k; + int rs_matC = bli_obj_row_stride( c ); + int rsc = 1; + double *A = a->buffer; // pointer to matrix A elements, stored in row major format + double *B = b->buffer; // pointer to matrix B elements, stored in column major format + double *C = D_C_pack; // pointer to matrix C elements, stored in column major format + double *matCbuf = c->buffer; + + double *tA = A, *tB = B, *tC = C; + + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm0, ymm1, ymm2, ymm3; + + double result, scratch[8]; + double *alpha_cast, *beta_cast; // alpha, beta multiples + alpha_cast = (alpha->buffer); + beta_cast = (beta->buffer); + + // The non-copy version of the A^T SYRK gives better performance for the small M cases. + // The threshold is controlled by BLIS_ATBN_M_THRES + if (M <= BLIS_ATBN_M_THRES) + { + for (col_idx = 0; (col_idx + (NR - 1)) < N; col_idx += NR) + { + for (row_idx = 0; (row_idx + (AT_MR - 1)) < M; row_idx += AT_MR) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + //The inner loop computes the 4x3 values of the matrix. + //The computation pattern is: + // ymm4 ymm5 ymm6 + // ymm7 ymm8 ymm9 + // ymm10 ymm11 ymm12 + // ymm13 ymm14 ymm15 + + //The Dot operation is performed in the inner loop, 4 double elements fit + //in the YMM register hence loop count incremented by 4 + for (k = 0; (k + 3) < K; k += 4) + { + ymm0 = _mm256_loadu_pd(tB + 0); + ymm1 = _mm256_loadu_pd(tB + ldb); + ymm2 = _mm256_loadu_pd(tB + 2 * ldb); + + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_pd(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_pd(ymm2, ymm3, ymm6); + + ymm3 = _mm256_loadu_pd(tA + lda); + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + ymm3 = _mm256_loadu_pd(tA + 2 * lda); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + ymm11 = _mm256_fmadd_pd(ymm1, ymm3, ymm11); + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 3 * lda); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + ymm14 = _mm256_fmadd_pd(ymm1, ymm3, ymm14); + ymm15 = _mm256_fmadd_pd(ymm2, ymm3, ymm15); + + tA += 4; + tB += 4; + + } + + // if K is not a multiple of 4, padding is done before load using temproary array. + if (k < K) + { + int iter; + double data_feeder[4] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_pd(data_feeder); + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter + ldb]; + ymm1 = _mm256_loadu_pd(data_feeder); + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter + 2 * ldb]; + ymm2 = _mm256_loadu_pd(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_pd(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_pd(ymm2, ymm3, ymm6); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[2 * lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + ymm11 = _mm256_fmadd_pd(ymm1, ymm3, ymm11); + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[3 * lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + ymm14 = _mm256_fmadd_pd(ymm1, ymm3, ymm14); + ymm15 = _mm256_fmadd_pd(ymm2, ymm3, ymm15); + + } + + //horizontal addition and storage of the data. + //Results for 4x3 blocks of C is stored here + ymm4 = _mm256_hadd_pd(ymm4, ymm4); + _mm256_storeu_pd(scratch, ymm4); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm7 = _mm256_hadd_pd(ymm7, ymm7); + _mm256_storeu_pd(scratch, ymm7); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm10 = _mm256_hadd_pd(ymm10, ymm10); + _mm256_storeu_pd(scratch, ymm10); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm13 = _mm256_hadd_pd(ymm13, ymm13); + _mm256_storeu_pd(scratch, ymm13); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + + tC += ldc; + ymm5 = _mm256_hadd_pd(ymm5, ymm5); + _mm256_storeu_pd(scratch, ymm5); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm8 = _mm256_hadd_pd(ymm8, ymm8); + _mm256_storeu_pd(scratch, ymm8); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm11 = _mm256_hadd_pd(ymm11, ymm11); + _mm256_storeu_pd(scratch, ymm11); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm14 = _mm256_hadd_pd(ymm14, ymm14); + _mm256_storeu_pd(scratch, ymm14); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + + tC += ldc; + ymm6 = _mm256_hadd_pd(ymm6, ymm6); + _mm256_storeu_pd(scratch, ymm6); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm9 = _mm256_hadd_pd(ymm9, ymm9); + _mm256_storeu_pd(scratch, ymm9); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm12 = _mm256_hadd_pd(ymm12, ymm12); + _mm256_storeu_pd(scratch, ymm12); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm15 = _mm256_hadd_pd(ymm15, ymm15); + _mm256_storeu_pd(scratch, ymm15); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + } + } + + int processed_col = col_idx; + int processed_row = row_idx; + + //The edge case handling where N is not a multiple of 3 + if (processed_col < N) + { + for (col_idx = processed_col; col_idx < N; col_idx += 1) + { + for (row_idx = 0; (row_idx + (AT_MR - 1)) < M; row_idx += AT_MR) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + + //The inner loop computes the 4x1 values of the matrix. + //The computation pattern is: + // ymm4 + // ymm7 + // ymm10 + // ymm13 + + for (k = 0; (k + 3) < K; k += 4) + { + ymm0 = _mm256_loadu_pd(tB + 0); + + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + ymm3 = _mm256_loadu_pd(tA + lda); + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + + ymm3 = _mm256_loadu_pd(tA + 2 * lda); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + + ymm3 = _mm256_loadu_pd(tA + 3 * lda); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + + tA += 4; + tB += 4; + } + // if K is not a multiple of 4, padding is done before load using temproary array. + if (k < K) + { + int iter; + double data_feeder[4] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_pd(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[2 * lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[3 * lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + + } + + //horizontal addition and storage of the data. + //Results for 4x1 blocks of C is stored here + ymm4 = _mm256_hadd_pd(ymm4, ymm4); + _mm256_storeu_pd(scratch, ymm4); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm7 = _mm256_hadd_pd(ymm7, ymm7); + _mm256_storeu_pd(scratch, ymm7); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm10 = _mm256_hadd_pd(ymm10, ymm10); + _mm256_storeu_pd(scratch, ymm10); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm13 = _mm256_hadd_pd(ymm13, ymm13); + _mm256_storeu_pd(scratch, ymm13); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + } + } + processed_row = row_idx; + } + + // The edge case handling where M is not a multiple of 4 + if (processed_row < M) + { + for (row_idx = processed_row; row_idx < M; row_idx += 1) + { + for (col_idx = 0; col_idx < N; col_idx += 1) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + + for (k = 0; (k + 3) < K; k += 4) + { + ymm0 = _mm256_loadu_pd(tB + 0); + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + tA += 4; + tB += 4; + } + + // if K is not a multiple of 4, padding is done before load using temproary array. + if (k < K) + { + int iter; + double data_feeder[4] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_pd(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + } + + //horizontal addition and storage of the data. + ymm4 = _mm256_hadd_pd(ymm4, ymm4); + _mm256_storeu_pd(scratch, ymm4); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + } + } + } + + //copy/compute sryk values back to C + if ( bli_seq0( *beta_cast ) ) //when beta is 0, just copy result to C + { + dim_t _i, _j; + if(bli_obj_is_lower(c)) //c is lower + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i <= 0 ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i >= 0 ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + } + else //when beta is non-zero, multiply and store result to C + { + dim_t _i, _j; + if(bli_obj_is_lower(c)) //c is lower + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i <= 0 ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i >= 0 ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + } + + return BLIS_SUCCESS; + } + else + return BLIS_NONCONFORMAL_DIMENSIONS; +} + +#endif + diff --git a/kernels/zen/3/bli_trsm_small.c b/kernels/zen/3/bli_trsm_small.c new file mode 100644 index 000000000..af84d0588 --- /dev/null +++ b/kernels/zen/3/bli_trsm_small.c @@ -0,0 +1,25101 @@ +/* + +BLIS +An object-based framework for developing high-performance BLAS-like +libraries. + +Copyright (C) 2018-2019, Advanced Micro Devices, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of The University of Texas at Austin nor the names +of its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" +#ifdef BLIS_ENABLE_SMALL_MATRIX_TRSM +#include "immintrin.h" +#define GEMM_BLK_V1 8 //Block size to perform gemm and apply trsm +#define GEMM_ACCUM_A 1 //Peform B1=B1-(B0*A0) operation instead of B1'=(B0*A0) and then B1=B1-B1' +#define OPT_CACHE_BLOCKING_L1 1 //Perform trsm block-wise in blocks of GEMM_BLK_V1 instead of all columns of B together. +#define REARRANGE_SHFL 0 //Rearrange operations using blend or shuffle +#define BLI_AlXB_M_SP 16 +#define BLI_XAltB_N_SP 128 +#define BLI_AutXB_M_SP 64 +#define BLI_AutXB_N_SP 128 + +// XA = B; A is lower-traingular; No transpose; double precision; non-unit diagonal +static err_t bli_dtrsm_small_XAlB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//XA = B; A is lower triabgular; No transpose; double precision; unit-diagonal +static err_t bli_dtrsm_small_XAlB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//XA = B; A is lower-triangular; A is transposed; double precision; non-unit-diagonal +static err_t bli_dtrsm_small_XAltB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//XA = B; A is lower-triangular; A is transposed; double precision; unit-diagonal +static err_t bli_dtrsm_small_XAltB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +// XA = B; A is upper triangular; No transpose; double presicion; non-unit diagonal +static err_t bli_dtrsm_small_XAuB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//XA = B; A is upper triangular; No transpose; double precision; unit-diagonal +static err_t bli_dtrsm_small_XAuB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//XA = B; A is upper-triangular; A is transposed; double precision; non-unit diagonal +static err_t bli_dtrsm_small_XAutB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//XA = B; A is upper-triangular; A is transposed; double precision; unit diagonal +static err_t bli_dtrsm_small_XAutB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//AX = B; A is lower triangular; No transpose; double precision; non-unit diagonal +static err_t bli_dtrsm_small_AlXB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//AX = B; A is lower triangular; No transpose; double precision; unit diagonal +static err_t bli_dtrsm_small_AlXB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + + + +static void (*fp_blis_strsm_microkernel)( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); +static void blis_strsm_microkernel( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); +static void blis_strsm_microkernel_alpha( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal + ); +static void blis_strsm_microkernel_unitDiag( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); +static void blis_strsm_microkernel_alpha_unitDiag( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal + ); +static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal); +static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal); + + +static void blis_dtrsm_microkernel( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); + +static void blis_dtrsm_microkernel_alpha( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ); + +static void blis_dtrsm_microkernel_unitDiag( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); + +static void blis_dtrsm_microkernel_alpha_unitDiag( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ); + +static void dtrsm_XAtB_block_allSmallSizedMatrices(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal); +static void dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal); +static void trsm_AutXB_block_allSmallSizedMatrices(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void trsm_AutXB_block_allSmallSizedMatrices_alpha(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alpha); +static void trsm_AutXB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alpha); + +//AX = B; A is lower triangular; No transpose; single precision +static err_t bli_strsm_small_AlXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); +//A.'X = B; A is upper triangular; A has to be transposed; single precision +static err_t bli_strsm_small_AutXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//XA.' = B; A is lower triangular; A has to be transposed; single precision +static err_t bli_strsm_small_XAltB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//A.'X = B; A is upper triangular; A has to be transposed; double precision +static err_t bli_dtrsm_small_AutXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +/* +* The bli_trsm_small implements unpacked version of TRSM +* Currently only column-major is supported, A & B are column-major +* Input: A: MxM (triangular matrix) +* B: MxN matrix +* Output: X: MxN matrix such that AX = alpha*B or XA = alpha*B or A'X = alpha*B or XA' = alpha*B +* Here the output X is stored in B +* The custom-kernel will be called only when M*(M+N)* sizeof(Matrix Elements) < L3 cache +*/ +err_t bli_trsm_small + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ +#ifdef BLIS_ENABLE_MULTITHREADING + return BLIS_NOT_YET_IMPLEMENTED; +#endif + + dim_t m = bli_obj_length(b); + dim_t n = bli_obj_width(b); + + if(!(m && n)) + return BLIS_SUCCESS; + + + // If alpha is zero, B matrix will become zero after scaling & hence solution is also zero matrix + if (bli_obj_equals(alpha, &BLIS_ZERO)) + { + return BLIS_NOT_YET_IMPLEMENTED; // scale B by alpha + } + // We have to call matrix scaling if alpha != 1.0 + + // if row major format return. Check this again. + if ((bli_obj_row_stride(a) != 1) || + (bli_obj_row_stride(b) != 1)) + { + return BLIS_INVALID_ROW_STRIDE; + } + + num_t dt = ((*b).info & (0x7 << 0)); + + // only float and double datatypes are supported as of now. + if (dt != BLIS_DOUBLE && dt != BLIS_FLOAT) + { + return BLIS_EXPECTED_REAL_DATATYPE; + } + + // A is expected to be triangular in trsm + if (!bli_obj_is_upper_or_lower (a)) + { + return BLIS_EXPECTED_TRIANGULAR_OBJECT; + } + + // can use other control structs - even can use array of function pointers, + // indexed by a number with bits formed by f('side', 'uplo', 'transa', dt). + // In the below implementation, based on the number of finally implemented + // cases, can move the checks with more cases higher up. + + if(side == BLIS_LEFT) + { + if(bli_obj_has_trans(a)) + { + if(dt == BLIS_DOUBLE) + { + if(bli_obj_is_upper(a)) + { + //return bli_dtrsm_small_AutXB(side, alpha, a, b, cntx, cntl); + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //return bli_dtrsm_small_AltXB(side, alpha, a, b, cntx, cntl); + return BLIS_NOT_YET_IMPLEMENTED; + } + } + else + { + if(bli_obj_is_upper(a)) + { + return bli_strsm_small_AutXB(side, alpha, a, b, cntx, cntl); + } + else + { + //return bli_strsm_small_AltXB(side, alpha, a, b, cntx, cntl); + return BLIS_NOT_YET_IMPLEMENTED; + } + + } + } + else + { + if(dt == BLIS_DOUBLE) + { + if(bli_obj_is_upper(a)) + { + //return bli_dtrsm_small_AuXB(side, alpha, a, b, cntx, cntl); + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + if(bli_obj_has_unit_diag(a)) + return bli_dtrsm_small_AlXB_unitDiag(side, alpha, a, b, cntx, cntl); + else + return bli_dtrsm_small_AlXB(side, alpha, a, b, cntx, cntl); + } + } + else + { + if(bli_obj_is_upper(a)) + { + //return bli_strsm_small_AuXB(side, alpha, a, b, cntx, cntl); + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + return bli_strsm_small_AlXB(side, alpha, a, b, cntx, cntl); + } + + } + + } + } + else + { + if(bli_obj_has_trans(a)) + { + if(dt == BLIS_DOUBLE) + { + if(bli_obj_is_upper(a)) + { + if(bli_obj_has_unit_diag(a)) + return bli_dtrsm_small_XAutB_unitDiag(side, alpha, a, b, cntx, cntl); + else + return bli_dtrsm_small_XAutB(side, alpha, a, b, cntx, cntl); + } + else + { + if(bli_obj_has_unit_diag(a)) + return bli_dtrsm_small_XAltB_unitDiag(side, alpha, a, b, cntx, cntl); + else + return bli_dtrsm_small_XAltB(side, alpha, a, b, cntx, cntl); + } + } + else + { + if(bli_obj_is_upper(a)) + { + //return bli_strsm_small_XAutB(side, alpha, a, b, cntx, cntl); + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + return bli_strsm_small_XAltB(side, alpha, a, b, cntx, cntl); + } + + } + } + else + { + if(dt == BLIS_DOUBLE) + { + if(bli_obj_is_upper(a)) + { + if(bli_obj_has_unit_diag(a)) + return bli_dtrsm_small_XAuB_unitDiag(side, alpha, a, b, cntx, cntl); + else + return bli_dtrsm_small_XAuB(side, alpha, a, b, cntx, cntl); + } + else + { + if(bli_obj_has_unit_diag(a)) + return bli_dtrsm_small_XAlB_unitDiag(side, alpha, a, b, cntx, cntl); + else + return bli_dtrsm_small_XAlB(side, alpha, a, b, cntx, cntl); + } + } + else + { + if(bli_obj_is_upper(a)) + { + //return bli_strsm_small_XAuB(side, alpha, a, b, cntx, cntl); + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //return bli_strsm_small_XAlB(side, alpha, a, b, cntx, cntl); + return BLIS_NOT_YET_IMPLEMENTED; + } + + } + + } + } + return BLIS_NOT_YET_IMPLEMENTED; +}; + +/* TRSM scalar code for the case AX = alpha * B + * A is lower-triangular, non-unit-diagonal, no transpose + * Dimensions: A: mxm X: mxn B:mxn + */ + +static err_t dtrsm_small_AlXB ( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb + ) +{ + + dim_t i, j, k; + + for (k = 0; k < M; k++) + { + double lkk_inv = 1.0/A[k+k*lda]; + for (j = 0; j < N; j++) + { + B[k + j*ldb] *= lkk_inv; + for (i = k+1; i < M; i++) + { + B[i + j*ldb] -= A[i + k*lda] * B[k + j*ldb]; + } + } + }// k -loop + return BLIS_SUCCESS; +}// end of function + +/* TRSM scalar code for the case AX = alpha * B + * A is lower-triangular, unit-diagonal, no transpose + * Dimensions: A: mxm X: mxn B:mxn + */ + +static err_t dtrsm_small_AlXB_unitDiag ( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb + ) +{ + + dim_t i, j, k; + + for (k = 0; k < M; k++) + { + for (j = 0; j < N; j++) + { + for (i = k+1; i < M; i++) + { + B[i + j*ldb] -= A[i + k*lda] * B[k + j*ldb]; + } + } + } + return BLIS_SUCCESS; +}// end of function + +/* TRSM scalar code for the case XA = alpha * B + * A is upper-triangular, non-unit-diagonal no transpose + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAuB ( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + for(k = 0; k < N; k++) + { + double lkk_inv = 1.0/A[k+k*lda]; + for(i = 0; i < M; i++) + { + B[i+k*ldb] *= lkk_inv; + for(j = k+1; j < N; j++) + { + B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; + } + } + + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is lower-triangular, non-unit triangular, no transpose + * Dimensions: X:mxn A:nxn B:mxn + */ + +static err_t dtrsm_small_XAlB ( + double *A, + double *B, + double alpha, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + for(i = 0; i < M; i++) + for(j = 0; j < N; j++) + B[i+j*ldb] *= alpha; + + for(k = N-1; k+1 > 0; k--) + { + double lkk_inv = 1.0/A[k+k*lda]; + for(i = M-1; i+1 > 0; i--) + { + B[i+k*ldb] *= lkk_inv; + for(j = k-1; j+1 > 0; j--) + { + B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is lower-triangular, unit-diagonal, no transpose + *Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAlB_unitDiag( + double *A, + double *B, + double alpha, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(i = 0 ; i < M; i++) + for(j = 0; j < N; j++) + B[i+j*ldb] *= alpha; + + for(k = N-1; k+1 > 0; k--) + { + for(i = M-1; i+1 > 0; i--) + { + for(j = k-1; j+1 > 0; j--) + { + B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + *A is upper-triangular, non-unit-diagonal, A is transposed + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAutB ( + double *A, + double *B, + double alpha, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(i = 0; i < M; i++) + for(j = 0; j < N; j++) + B[i+j*ldb] *=alpha; + + for(k = N-1; k+1 > 0; k--) + { + double lkk_inv = 1.0/A[k+k*lda]; + for(i = M-1; i+1 > 0; i--) + { + B[i+k*ldb] *= lkk_inv; + for(j = k-1; j+1 > 0; j--) + { + B[i+j*ldb] -= B[i+k*ldb] * A[j+k*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is upper-triangular, unit-diagonal, A has to be transposed + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAutB_unitDiag( + double *A, + double *B, + double alpha, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(i = 0; i< M; i++) + for(j = 0; j< N; j++) + B[i+j*ldb] *= alpha; + + for(i = M-1; i+1 > 0; i--) + { + for(j = N-1; j+1 > 0; j--) + { + for(k = j-1; k+1 > 0; k--) + { + B[i+k*ldb] -= B[i+j*ldb] * A[k+j*lda]; + + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is lower-triangular, non-unit-diagonal, A has to be transposed + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAltB ( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(k = 0; k < N; k++) + { + double lkk_inv = 1.0/A[k+k*lda]; + for(i = 0; i < M; i++) + { + B[i+k*ldb] *= lkk_inv; + for(j = k+1; j < N; j++) + { + B[i+j*ldb] -= B[i+k*ldb] * A[j+k*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for XA = alpha * B + * A is lower-triangular, unit-diagonal, A has to be transposed + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAltB_unitDiag( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(k = 0; k < N; k++) + { + for(i = 0; i < M; i++) + { + for(j = k+1; j < N; j++) + { + B[i+j*ldb] -= B[i+k*ldb] * A[j+k*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is upper-triangular, unit-diagonal, no transpose + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAuB_unitDiag ( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(k = 0; k < N; k++) + { + for(i = 0; i < M; i++) + { + for(j = k+1; j < N; j++) + { + B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM for the case AX = alpha * B, Double precision + * A is lower-triangular, no-transpose, non-unit diagonal + * dimensions A: mxm X: mxn B: mxn + + b01---> + * ***************** + ** * * * * * + * * * * * * * + * * *b01* * * * + * * * * * * * +a10 ****** b11 ***************** + | * * * | * * * * * + | * * * | * * * * * + | *a10*a11* | *b11* * * * + v * * * v * * * * * + *********** ***************** + * * * * * * * * * + * * * * * * * * * + * * * * * * * * * + * * * * * * * * * + **************** ***************** + a11---> +*/ + +static err_t bli_dtrsm_small_AlXB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + dim_t D_MR = 4; //size of block along 'M' dimpension + dim_t D_NR = 8; //size of block along 'N' dimension + + dim_t m = bli_obj_length(b); // number of rows of matrix B + dim_t n = bli_obj_width(b); // number of columns of matrix B + +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ALXB_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ALXB_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif + + dim_t m_remainder = m % D_MR; //number of remainder rows + dim_t n_remainder = n % D_NR; //number of remainder columns + + dim_t cs_a = bli_obj_col_stride(a); // column stride of A + dim_t cs_b = bli_obj_col_stride(b); // column stride of B + + dim_t i, j, k; //loop variables + dim_t k_iter; //number of times GEMM to be performed + + double AlphaVal = *(double *)AlphaObj->buffer; //value of alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a10, *a11, *b01, *b11; //pointers that point to blocks for GEMM and TRSM + double *ptr_b01_dup; + + double ones = 1.0; + + //scratch registers + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + + + for(j = 0; j+D_NR-1 < n; j += D_NR) //loop along 'N' dimension + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' dimension + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM to be performed(in blocks of 4x4) + + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + ///GEMM code begins/// + + for(k = 0; k< k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm16 = _mm256_loadu_pd((double const *)(a10));//A10[0][0] A10[1][0] A10[2][0] A10[3][0] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a));//A10[0][1] A10[1][1] A10[2][1] A10[3][1] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2));//A10[0][2] A10[1][2] A10[2][2] A10[3][2] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][4]*A10[3][2]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][5]*A10[3][2]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][6]*A10[3][2]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][7]*A10[3][2]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3));//A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[3][0] B01[3][0]*A10[2][3] B01[3][0]*A10[3][0]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[3][1]*A10[0][3] B01[3][1]*A10[3][0] B01[3][1]*A10[2][3] B01[3][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[3][2]*A10[0][3] B01[3][2]*A10[3][0] B01[3][2]*A10[2][3] B01[3][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[3][3]*A10[0][3] B01[3][3]*A10[3][0] B01[3][3]*A10[2][3] B01[3][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[3][4]*A10[0][3] B01[3][4]*A10[3][0] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[3][5]*A10[0][3] B01[3][5]*A10[3][0] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[3][6]*A10[0][3] B01[3][6]*A10[3][0] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[3][7]*A10[0][3] B01[3][7]*A10[3][0] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to calculate next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to calculate next block of B for GEMM + } + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha + + ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] * alpha -= B01[0-3][0] + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] * alpha -= B01[0-3][1] + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] * alpha -= B01[0-3][2] + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] * alpha -= B01[0-3][3] + ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] * alpha -= B01[0-3][4] + ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] * alpha -= B01[0-3][5] + ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] * alpha -= B01[0-3][6] + ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] * alpha -= B01[0-3][7] + + ///implement TRSM/// + + ///transpose of B11// + ///unpacklow/// + ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[2][4] B11[2][5] + ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[2][6] B11[2][7] + + //rearrange low elements + ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] + ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[1][4] B11[1][5] B11[3][4] B11[3][5] + ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[1][6] B11[1][7] B11[3][6] B11[3][7] + + //rearrange high elements + ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] + ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] + + ymm0 = _mm256_broadcast_sd((double const *)&ones); + + //broadcast diagonal elements of A11 + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] + ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] + + ymm5 = _mm256_unpacklo_pd(ymm1, ymm2); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] + ymm6 = _mm256_unpacklo_pd(ymm3, ymm4); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] + + ymm5 = _mm256_blend_pd(ymm5, ymm6, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm0 = _mm256_div_pd(ymm0, ymm5); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2] + + //extract a00 + ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + + //(Row 0): perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm1); //B11[0-3][0] /= A11[0][0] + ymm12 = _mm256_mul_pd(ymm12, ymm1); //B11[0-3][4] /= A11[0][0] + + //extract a11 + ymm1 = _mm256_permute_pd(ymm0, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] + + a11 += cs_a; + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= A11[1][0] * B11[0-3][0] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= A11[2][0] * B11[0-3][0] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= A11[3][0] * B11[0-3][0] + + ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= A11[1][0] * B11[0-3][4] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= A11[2][0] * B11[0-3][4] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= A11[3][0] * B11[0-3][4] + + ymm9 = _mm256_mul_pd(ymm9, ymm1); //B11[0-3][1] /= A11[1][1] + ymm13 = _mm256_mul_pd(ymm13, ymm1); //B11[0-3][5] /= A11[1][1] + + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] + + a11 += cs_a; + + //extract a22 + ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A110[][0] 1/A11[2][2] 1/A11[2][2] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(ROw2): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] + + ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] + + //perform mul operation + ymm10 = _mm256_mul_pd(ymm10, ymm1); //B11[0-3][2] /= A11[2][2] + ymm14 = _mm256_mul_pd(ymm14, ymm1); //B11[0-3][6] /= A11[2][2] + + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] + + a11 += cs_a; + + //extract a33 + ymm1 = _mm256_permute_pd(ymm0, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11);//1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(ROw2): FMA operations + ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[3][0-3] -= A11[3][2] * B11[0-3][2] + + ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[7][0-3] -= A11[3][2] * B11[0-3][6] + + //perform mul operation + ymm11 = _mm256_mul_pd(ymm11, ymm1); //B11[0-3][3] /= A11[3][3] + ymm15 = _mm256_mul_pd(ymm15, ymm1); //B11[0-3][7] /= A11[3][3] + + //unpacklow// + ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] + ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + + ///unpack high/// + ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[4][1] B11[5][1] B11[4][3] B11[5][3] + ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[6][1] B11[7][1] B11[6][3] B11[7][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store B11[0][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store B11[1][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store B11[2][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store B11[3][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store B11[4][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store B11[5][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store B11[6][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store B11[7][0-3] + } + + if(m_remainder) //implementation for reamainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM operation to be done(in blocks of 4x4) + + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + ///GEMM code Begins/// + for(k = 0; k< k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm16 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0] ) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm16 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 1)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] + + b01 += 1; //move to next row of B01 + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] //A10[1][2] A10[2][2] A10[3][2] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][0]*A10[3][2]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][1]*A10[3][2]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][2]*A10[3][2]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm8 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm8 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm8 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm8 += (B01[3][0]*A10[0][3] B01[3][4]*A10[1][3] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm8 += (B01[3][1]*A10[0][3] B01[3][5]*A10[1][3] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm8 += (B01[3][2]*A10[0][3] B01[3][6]*A10[1][3] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm8 += (B01[3][3]*A10[0][3] B01[3][7]*A10[1][3] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + } + + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value + + ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] *alpha -= B01[0-3][0] + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] *alpha -= B01[0-3][1] + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] *alpha -= B01[0-3][2] + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] *alpha -= B01[0-3][3] + ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] *alpha -= B01[0-3][4] + ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] *alpha -= B01[0-3][5] + ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] *alpha -= B01[0-3][6] + ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] *alpha -= B01[0-3][7] + + ///implement TRSM/// + + ///unpacklow/// + ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[1][4] B11[1][5] + ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[1][6] B11[1][7] + + //rearrange low elements + ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] + ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[5][0] B11[5][1] B11[7][0] B11[7][1] + ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[5][2] B11[5][3] B11[7][2] B11[7][3] + + //rearrange high elements + ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] + ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] + + + ymm0 = _mm256_broadcast_sd((double const *)&ones); + + //broadcast diagonal elements of A11 + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] + ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] + + ymm5 = _mm256_unpacklo_pd(ymm1, ymm2); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] + ymm6 = _mm256_unpacklo_pd(ymm3, ymm4); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] + + ymm5 = _mm256_blend_pd(ymm5, ymm6, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm0 = _mm256_div_pd(ymm0, ymm5); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a00 + ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + + //(Row 0): perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm1); //B11[0-3][0] /= A11[0][0] + ymm12 = _mm256_mul_pd(ymm12, ymm1); //B11[0-3][4] /= A11[0][0] + + //extract a11 + ymm1 = _mm256_permute_pd(ymm0, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] + + a11 += cs_a; + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= B11[0-3][0]*A11[1][0] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= B11[0-3][0]*A11[2][0] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= B11[0-3][0]*A11[3][0] + + ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= B11[0-3][4]*A11[1][4] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= B11[0-3][4]*A11[2][4] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= B11[0-3][4]*A11[3][4] + + ymm9 = _mm256_mul_pd(ymm9, ymm1); //B11[0-3][1] /= A11[1][1] + ymm13 = _mm256_mul_pd(ymm13, ymm1); //B11[0-3][5] /= A11[1][1] + + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] + + a11 += cs_a; + + //extract a22 + ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(ROw2): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] + + ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] + + //perform mul operation + ymm10 = _mm256_mul_pd(ymm10, ymm1); //B11[0-3][2] /=A11[2][2] + ymm14 = _mm256_mul_pd(ymm14, ymm1); //B11[0-3][6] /= A11[2][2] + + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] + + a11 += cs_a; + + //extract a33 + ymm1 = _mm256_permute_pd(ymm0, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(ROw2): FMA operations + ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[0-3][3] -= A11[3][2]*B11[0-3][2] + + ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[0-3][7] -= A11[3][2]*B11[0-3][6] + + //perform mul operation + ymm11 = _mm256_mul_pd(ymm11, ymm1); //B11[0-3][3] /= A11[3][3] + ymm15 = _mm256_mul_pd(ymm15, ymm1); //B11[0-3][7] /= A11[3][3] + + //unpacklow// + ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] + ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + + ///unpack high/// + ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[0][5] B11[1][5] B11[0][7] B11[1][7] + ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[2][5] B11[3][5] B11[2][7] B11[3][7] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm8 = _mm256_loadu_pd((double const *)(b11 + cs_b * 0)); //load B11[0-3][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b * 1)); //load B11[0-3][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //load B11[0-3][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //load B11[0-3][3] + ymm12 = _mm256_loadu_pd((double const *)(b11 + cs_b * 4)); //load B11[0-3][4] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b * 5)); //load B11[0-3][5] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b * 6)); //load B11[0-3][6] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b * 7)); //load B11[0-3][7] + //determine correct values to store + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x08); + ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x08); + ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x08); + ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x08); + ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x08); + ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x08); + ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x08); + ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x08); + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm0, ymm8, 0x30); + ymm1 = _mm256_permute2f128_pd(ymm1, ymm9, 0x30); + ymm2 = _mm256_permute2f128_pd(ymm2, ymm10, 0x30); + ymm3 = _mm256_permute2f128_pd(ymm3, ymm11, 0x30); + ymm4 = _mm256_permute2f128_pd(ymm4, ymm12, 0x30); + ymm5 = _mm256_permute2f128_pd(ymm5, ymm13, 0x30); + ymm6 = _mm256_permute2f128_pd(ymm6, ymm14, 0x30); + ymm7 = _mm256_permute2f128_pd(ymm7, ymm15, 0x30); + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x0E); + ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x0E); + ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x0E); + ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x0E); + ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x0E); + ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x0E); + ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x0E); + ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x0E); + } + + _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store(B11[0-3][4]) + _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store(B11[0-3][5]) + _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store(B11[0-3][6]) + _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store(B11[0-3][7]) + + } + } + + if((n & 4)) //implementation for remainder columns(when 'N' is a multiple of 4) + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM to be performed(in block of 4) + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b*3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a*2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[1][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[2][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[3][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B01[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B01[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B01[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + //compute reciprocals of L(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[0][0] A11[2][2] A11[2][2] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[1][1] A11[1][1] A11[3][3] A11[3][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + ymm4 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm4); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] +/* + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); +*/ + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] +/* + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); +*/ + //extract a00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm4 = _mm256_mul_pd(ymm4, ymm15); //B11[0][0-3] /= A11[0][0] + + //extract diag a11 from a + ymm15 = _mm256_permute_pd(ymm14, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]*B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]*B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]*B11[0][0-3] + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm15); //B11[1][0-3] /= A11[1][1] + + + //extract diag a22 from a + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]*B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]*B11[1][0-3] + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + ymm11 = _mm256_mul_pd(ymm11, ymm15); //B11[2][0-3] /= A11[2][2] + + //extract diag a33 from a + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]*B11[2][0-3] + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + ymm13 = _mm256_mul_pd(ymm13, ymm15); //B11[3][0-3] /= A11[3][3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b*2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[0-3][3]) + + } + if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + + k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) //looop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + //compute reciprocals of L(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + ymm4 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm4); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] +/* + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); +*/ + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] +/* + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); +*/ + //extract a00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00);//1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm4 = _mm256_mul_pd(ymm4, ymm15); //B11[0][0-3] /= A11[0][0] + + //extract diag a11 from a + ymm15 = _mm256_permute_pd(ymm14, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[][] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]* B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]* B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]* B11[0][0-3] + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm15); //B11[1][0-3] /= A11[1][1] + + + //extract diag a22 from a + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]* B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]* B11[1][0-3] + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + ymm11 = _mm256_mul_pd(ymm11, ymm15); //B11[2][0-3] /= A11[2][2] + + //extract diag a33 from a + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]* B11[2][0-3] + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + ymm13 = _mm256_mul_pd(ymm13, ymm15); //B11[3][0-3] /= A11[3][3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //load 4x4 block from b11 + ymm4 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][2] B11[3][3] + + //determine correct values to store + + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x08); + ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x08); + ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x08); + ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x08); + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm0, ymm4,0x30); + ymm1 = _mm256_permute2f128_pd(ymm1, ymm5,0x30); + ymm2 = _mm256_permute2f128_pd(ymm2, ymm6,0x30); + ymm3 = _mm256_permute2f128_pd(ymm3, ymm7,0x30); + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x0E); + ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x0E); + ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x0E); + ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x0E); + } + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) + + } + + n_remainder -= 4; + j += 4; + + } + + if(n_remainder) //implementation fo remaining columns(when 'N' is not a multiple of D_NR) + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha Value + + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + if(n_remainder == 3) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_broadcast_sd((double const *)&ones); + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const*)&ones); + } + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + } + + ///GEMM code ends/// + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + //compute reciprocals of L(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + ymm4 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm4); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] +/* + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); +*/ + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] +/* + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); +*/ + //extract a00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm4 = _mm256_mul_pd(ymm4, ymm15); //B11[0][0-3] /= A11[0][0] + + //extract diag a11 from a + ymm15 = _mm256_permute_pd(ymm14, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0] * B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0] * B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0] * B11[0][0-3] + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm15); //B11[1][0-3] /= A11[1][1] + + + //extract diag a22 from a + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1] * B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1] * B11[1][0-3] + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + ymm11 = _mm256_mul_pd(ymm11, ymm15); //B11[2][0-3] /= A11[2][2] + + //extract diag a33 from a + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2] * B11[2][0-3] + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + ymm13 = _mm256_mul_pd(ymm13, ymm15); //B11[3][0-3] /= A11[3][3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + } + + } + if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + + k_iter = i / D_MR; //number of times GEMM operations to be performed + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha value + + ///GEMM for previously calculated values /// + + + //load 4x4 block from b11 + if(n_remainder == 3) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_broadcast_sd((double const *)&ones); + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[0][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[1][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[2][3] + + b01 += 1; //move to next row of B + + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[3][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm8 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] * alpha -= ymm4 + ymm9 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] * alpha -= ymm5 + ymm10 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] * alpha -= ymm6 + ymm11 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] * alpha -= ymm7 + + ///implement TRSM/// + //determine correct values to store + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x08); + ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x08); + ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x08); + ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x08); + + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm8, ymm0, 0x30); + ymm1 = _mm256_permute2f128_pd(ymm9, ymm1, 0x30); + ymm2 = _mm256_permute2f128_pd(ymm10, ymm2, 0x30); + ymm3 = _mm256_permute2f128_pd(ymm11, ymm3, 0x30); + + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x0E); + ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x0E); + ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x0E); + ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x0E); + } + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + } + + ///scalar code for trsm without alpha/// + dtrsm_small_AlXB(a11, b11, m_remainder, n_remainder, cs_a, cs_b); + } + } + return BLIS_SUCCESS; +} + +/* TRSM for the case AX = alpha * B, Double precision + * A is lower-triangular, no-transpose, unit diagonal + * dimensions A: mxm X: mxn B: mxn + + b01---> + * ***************** + ** * * * * * + * * * * * * * + * * *b01* * * * + * * * * * * * +a10 ****** b11 ***************** + | * * * | * * * * * + | * * * | * * * * * + | *a10*a11* | *b11* * * * + v * * * v * * * * * + *********** ***************** + * * * * * * * * * + * * * * * * * * * + * * * * * * * * * + * * * * * * * * * + **************** ***************** + a11---> +*/ + +static err_t bli_dtrsm_small_AlXB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + dim_t D_MR = 4; //size of block along 'M' dimpension + dim_t D_NR = 8; //size of block along 'N' dimension + + dim_t m = bli_obj_length(b); // number of rows of matrix B + dim_t n = bli_obj_width(b); // number of columns of matrix B + +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ALXB_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ALXB_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif + + dim_t m_remainder = m % D_MR; //number of remainder rows + dim_t n_remainder = n % D_NR; //number of remainder columns + + dim_t cs_a = bli_obj_col_stride(a); // column stride of A + dim_t cs_b = bli_obj_col_stride(b); // column stride of B + + dim_t i, j, k; //loop variables + dim_t k_iter; //number of times GEMM to be performed + + double AlphaVal = *(double *)AlphaObj->buffer; //value of alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a10, *a11, *b01, *b11; //pointers that point to blocks for GEMM and TRSM + double *ptr_b01_dup; + + double ones = 1.0; + + //scratch registers + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + + + for(j = 0; j+D_NR-1 < n; j += D_NR) //loop along 'N' dimension + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' dimension + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM to be performed(in blocks of 4x4) + + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + ///GEMM code begins/// + + for(k = 0; k< k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm16 = _mm256_loadu_pd((double const *)(a10));//A10[0][0] A10[1][0] A10[2][0] A10[3][0] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a));//A10[0][1] A10[1][1] A10[2][1] A10[3][1] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2));//A10[0][2] A10[1][2] A10[2][2] A10[3][2] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][4]*A10[3][2]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][5]*A10[3][2]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][6]*A10[3][2]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][7]*A10[3][2]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3));//A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[3][0] B01[3][0]*A10[2][3] B01[3][0]*A10[3][0]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[3][1]*A10[0][3] B01[3][1]*A10[3][0] B01[3][1]*A10[2][3] B01[3][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[3][2]*A10[0][3] B01[3][2]*A10[3][0] B01[3][2]*A10[2][3] B01[3][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[3][3]*A10[0][3] B01[3][3]*A10[3][0] B01[3][3]*A10[2][3] B01[3][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[3][4]*A10[0][3] B01[3][4]*A10[3][0] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[3][5]*A10[0][3] B01[3][5]*A10[3][0] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[3][6]*A10[0][3] B01[3][6]*A10[3][0] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[3][7]*A10[0][3] B01[3][7]*A10[3][0] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to calculate next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to calculate next block of B for GEMM + } + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha + + ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] * alpha -= B01[0-3][0] + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] * alpha -= B01[0-3][1] + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] * alpha -= B01[0-3][2] + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] * alpha -= B01[0-3][3] + ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] * alpha -= B01[0-3][4] + ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] * alpha -= B01[0-3][5] + ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] * alpha -= B01[0-3][6] + ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] * alpha -= B01[0-3][7] + + ///implement TRSM/// + + ///transpose of B11// + ///unpacklow/// + ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[2][4] B11[2][5] + ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[2][6] B11[2][7] + + //rearrange low elements + ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] + ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[1][4] B11[1][5] B11[3][4] B11[3][5] + ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[1][6] B11[1][7] B11[3][6] B11[3][7] + + //rearrange high elements + ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] + ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] + + //broadcast diagonal elements of A11 + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] + ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] + + a11 += cs_a; + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= A11[1][0] * B11[0-3][0] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= A11[2][0] * B11[0-3][0] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= A11[3][0] * B11[0-3][0] + + ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= A11[1][0] * B11[0-3][4] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= A11[2][0] * B11[0-3][4] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= A11[3][0] * B11[0-3][4] + + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] + + a11 += cs_a; + + //(ROw2): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] + + ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] + + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] + + a11 += cs_a; + + //(ROw2): FMA operations + ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[3][0-3] -= A11[3][2] * B11[0-3][2] + + ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[7][0-3] -= A11[3][2] * B11[0-3][6] + + //unpacklow// + ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] + ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + + ///unpack high/// + ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[4][1] B11[5][1] B11[4][3] B11[5][3] + ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[6][1] B11[7][1] B11[6][3] B11[7][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store B11[0][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store B11[1][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store B11[2][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store B11[3][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store B11[4][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store B11[5][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store B11[6][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store B11[7][0-3] + } + + if(m_remainder) //implementation for reamainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM operation to be done(in blocks of 4x4) + + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + ///GEMM code Begins/// + for(k = 0; k< k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm16 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0] ) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm16 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 1)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] + + b01 += 1; //move to next row of B01 + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] //A10[1][2] A10[2][2] A10[3][2] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][0]*A10[3][2]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][1]*A10[3][2]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][2]*A10[3][2]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm8 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm8 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm8 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm8 += (B01[3][0]*A10[0][3] B01[3][4]*A10[1][3] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm8 += (B01[3][1]*A10[0][3] B01[3][5]*A10[1][3] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm8 += (B01[3][2]*A10[0][3] B01[3][6]*A10[1][3] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm8 += (B01[3][3]*A10[0][3] B01[3][7]*A10[1][3] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + } + + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value + + ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] *alpha -= B01[0-3][0] + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] *alpha -= B01[0-3][1] + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] *alpha -= B01[0-3][2] + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] *alpha -= B01[0-3][3] + ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] *alpha -= B01[0-3][4] + ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] *alpha -= B01[0-3][5] + ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] *alpha -= B01[0-3][6] + ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] *alpha -= B01[0-3][7] + + ///implement TRSM/// + + ///unpacklow/// + ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[1][4] B11[1][5] + ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[1][6] B11[1][7] + + //rearrange low elements + ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] + ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[5][0] B11[5][1] B11[7][0] B11[7][1] + ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[5][2] B11[5][3] B11[7][2] B11[7][3] + + //rearrange high elements + ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] + ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] + + //broadcast diagonal elements of A11 + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] + ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] + + a11 += cs_a; + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= B11[0-3][0]*A11[1][0] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= B11[0-3][0]*A11[2][0] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= B11[0-3][0]*A11[3][0] + + ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= B11[0-3][4]*A11[1][4] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= B11[0-3][4]*A11[2][4] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= B11[0-3][4]*A11[3][4] + + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] + + a11 += cs_a; + + //(ROw2): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] + + ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] + + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] + + a11 += cs_a; + + //(ROw2): FMA operations + ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[0-3][3] -= A11[3][2]*B11[0-3][2] + + ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[0-3][7] -= A11[3][2]*B11[0-3][6] + + //unpacklow// + ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] + ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + + ///unpack high/// + ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[0][5] B11[1][5] B11[0][7] B11[1][7] + ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[2][5] B11[3][5] B11[2][7] B11[3][7] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm8 = _mm256_loadu_pd((double const *)(b11 + cs_b * 0)); //load B11[0-3][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b * 1)); //load B11[0-3][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //load B11[0-3][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //load B11[0-3][3] + ymm12 = _mm256_loadu_pd((double const *)(b11 + cs_b * 4)); //load B11[0-3][4] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b * 5)); //load B11[0-3][5] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b * 6)); //load B11[0-3][6] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b * 7)); //load B11[0-3][7] + //determine correct values to store + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x08); + ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x08); + ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x08); + ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x08); + ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x08); + ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x08); + ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x08); + ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x08); + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm0, ymm8, 0x30); + ymm1 = _mm256_permute2f128_pd(ymm1, ymm9, 0x30); + ymm2 = _mm256_permute2f128_pd(ymm2, ymm10, 0x30); + ymm3 = _mm256_permute2f128_pd(ymm3, ymm11, 0x30); + ymm4 = _mm256_permute2f128_pd(ymm4, ymm12, 0x30); + ymm5 = _mm256_permute2f128_pd(ymm5, ymm13, 0x30); + ymm6 = _mm256_permute2f128_pd(ymm6, ymm14, 0x30); + ymm7 = _mm256_permute2f128_pd(ymm7, ymm15, 0x30); + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x0E); + ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x0E); + ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x0E); + ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x0E); + ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x0E); + ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x0E); + ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x0E); + ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x0E); + } + + _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store(B11[0-3][4]) + _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store(B11[0-3][5]) + _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store(B11[0-3][6]) + _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store(B11[0-3][7]) + + } + } + + if((n & 4)) //implementation for remainder columns(when 'N' is a multiple of 4) + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM to be performed(in block of 4) + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b*3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a*2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[1][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[2][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[3][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B01[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B01[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B01[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]*B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]*B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]*B11[0][0-3] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]*B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]*B11[1][0-3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]*B11[2][0-3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b*2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[0-3][3]) + + } + if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + + k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) //looop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]* B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]* B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]* B11[0][0-3] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]* B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]* B11[1][0-3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]* B11[2][0-3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //load 4x4 block from b11 + ymm4 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][2] B11[3][3] + + //determine correct values to store + + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x08); + ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x08); + ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x08); + ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x08); + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm0, ymm4,0x30); + ymm1 = _mm256_permute2f128_pd(ymm1, ymm5,0x30); + ymm2 = _mm256_permute2f128_pd(ymm2, ymm6,0x30); + ymm3 = _mm256_permute2f128_pd(ymm3, ymm7,0x30); + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x0E); + ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x0E); + ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x0E); + ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x0E); + } + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) + + } + + n_remainder -= 4; + j += 4; + + } + + if(n_remainder) //implementation fo remaining columns(when 'N' is not a multiple of D_NR) + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha Value + + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + if(n_remainder == 3) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_broadcast_sd((double const *)&ones); + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const*)&ones); + } + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + } + + ///GEMM code ends/// + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0] * B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0] * B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0] * B11[0][0-3] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1] * B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1] * B11[1][0-3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2] * B11[2][0-3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + } + + } + if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + + k_iter = i / D_MR; //number of times GEMM operations to be performed + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha value + + ///GEMM for previously calculated values /// + + + //load 4x4 block from b11 + if(n_remainder == 3) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_broadcast_sd((double const *)&ones); + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[0][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[1][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[2][3] + + b01 += 1; //move to next row of B + + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[3][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm8 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] * alpha -= ymm4 + ymm9 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] * alpha -= ymm5 + ymm10 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] * alpha -= ymm6 + ymm11 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] * alpha -= ymm7 + + ///implement TRSM/// + //determine correct values to store + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x08); + ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x08); + ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x08); + ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x08); + + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm8, ymm0, 0x30); + ymm1 = _mm256_permute2f128_pd(ymm9, ymm1, 0x30); + ymm2 = _mm256_permute2f128_pd(ymm10, ymm2, 0x30); + ymm3 = _mm256_permute2f128_pd(ymm11, ymm3, 0x30); + + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x0E); + ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x0E); + ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x0E); + ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x0E); + } + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + } + + ///scalar code for trsm without alpha/// + dtrsm_small_AlXB_unitDiag(a11, b11, m_remainder, n_remainder, cs_a, cs_b); + } + } + return BLIS_SUCCESS; +} + + +/*implements TRSM for the case XA = alpha * B + *A is upper triangular, non-unit diagonal, no transpose + *dimensions: X:mxn A:nxn B: mxn + */ + +/* b11---> a01 ----> + ***************** *********** + *b01*b11* * * * * * * +b11 * * * * * **a01 * * a11 + | ***************** ********* | + | * * * * * *a11* * | + | * * * * * * * * | + v ***************** ****** v + * * * * * * * + * * * * * * * + ***************** * * + * + +*/ +static err_t bli_dtrsm_small_XAuB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n)>D_BLIS_SMALL_MATRIX_THRES_TRSM_XAUB_ROME && (m/n) < D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n)>D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES && (m/n) < D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction + { + for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction + { + a01 = L + j*cs_a; //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used in GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] + + //4th col + a11 += cs_a; + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] + + ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] + + ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j*cs_a; //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 2) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 1) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] + + //4th col + a11 += cs_a; + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] + + ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] + + ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + } + } + } + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = 0; (j+D_NR-1) a01 ----> + ***************** *********** + *b01*b11* * * * * * * +b11 * * * * * **a01 * * a11 + | ***************** ********* | + | * * * * * *a11* * | + | * * * * * * * * | + v ***************** ****** v + * * * * * * * + * * * * * * * + ***************** * * + * + +*/ + +static err_t bli_dtrsm_small_XAuB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n)>D_BLIS_SMALL_MATRIX_THRES_TRSM_XAUB_ROME && (m/n) < D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n)>D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES && (m/n) < D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction + { + for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction + { + a01 = L + j*cs_a; //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used in GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] + + //4th col + a11 += cs_a; + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j*cs_a; //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 2) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 1) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] + + //4th col + a11 += cs_a; + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] + + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + } + } + } + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = 0; (j+D_NR-1) a01 ----> + ***************** *********** + *b01*b11* * * * * * * +b11 * * * * * **a01 * * a11 + | ***************** ********* | + | * * * * * *a11* * | + | * * * * * * * * | + v ***************** ****** v + * * * * * * * + * * * * * * * + ***************** * * + * + +*/ +static err_t bli_dtrsm_small_XAltB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction + { + for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction + { + a01 = L + j; //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used in GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] + + ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] + + ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j; //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 2) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 1) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] + + ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] + + ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + } + } + } + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = 0; (j+D_NR-1) a01 ----> + ***************** *********** + *b01*b11* * * * * * * +b11 * * * * * **a01 * * a11 + | ***************** ********* | + | * * * * * *a11* * | + | * * * * * * * * | + v ***************** ****** v + * * * * * * * + * * * * * * * + ***************** * * + * + +*/ +static err_t bli_dtrsm_small_XAltB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction + { + for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction + { + a01 = L + j; //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used in GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j; //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 2) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 1) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + } + } + } + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = 0; (j+D_NR-1) D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction + { + for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction + { + a01 = L + j*cs_a +(j+D_NR); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm11 = _mm256_mul_pd(ymm11, ymm0); + + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(row 3):FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + ymm10 = _mm256_mul_pd(ymm10, ymm0); + + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + ymm9 = _mm256_mul_pd(ymm9, ymm0); + + ymm13 = _mm256_mul_pd(ymm13, ymm0); + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1): FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + + + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j + D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] + } + if(n_remainder == 2) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] + } + if(n_remainder == 1) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm11 = _mm256_mul_pd(ymm11, ymm0); + + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(row 3):FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + ymm10 = _mm256_mul_pd(ymm10, ymm0); + + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + ymm9 = _mm256_mul_pd(ymm9, ymm0); + + ymm13 = _mm256_mul_pd(ymm13, ymm0); + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1): FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + } + } + if(i<0) + i += D_NR; + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a33 + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm3 = _mm256_mul_pd(ymm3, ymm15); + + //extract a22 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + ymm2 = _mm256_mul_pd(ymm2, ymm15); + + //extract a11 + ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + ymm1 = _mm256_mul_pd(ymm1, ymm15); + + //extract A00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + ymm0 = _mm256_mul_pd(ymm0, ymm15); + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointwr to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + if(n_remainder == 3) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + } + if(n_remainder == 2) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + } + if(n_remainder == 1) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + } + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm16); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm16); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm16); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm16); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + + ///GEMM processing stars/// + + for(k = 0; k < k_iter; k++) + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + + } + + ///GEMM code ends/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -= ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a33 + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm3 = _mm256_mul_pd(ymm3, ymm15); + + //extract a22 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + ymm2 = _mm256_mul_pd(ymm2, ymm15); + + //extract a11 + ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + ymm1 = _mm256_mul_pd(ymm1, ymm15); + + //extract A00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + ymm0 = _mm256_mul_pd(ymm0, ymm15); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) + } + + } + m_remainder -= 4; + i -= 4; + } +// if(i < 0) i = 0; + if(m_remainder) ///implementation for remainder rows + { + dtrsm_small_XAlB(L, B, AlphaVal, m_remainder, n, cs_a, cs_b); + } + return BLIS_SUCCESS; +} + +/*implements TRSM for the case XA = alpha * B + *A is lower triangular, unit-diagonal, no transpose + *dimensions: X:mxn A:nxn B: mxn + */ + +/* <---b11 <---a11 + ***************** * + *b01*b11* * * * * + ^ * * * * * ^ * * + | ***************** | ******* + | * * * * * | * * * + | * * * * * a01* * * +b10 ***************** ************* + * * * * * * * * * + * * * * * * * * * + ***************** ******************* + +*/ +static err_t bli_dtrsm_small_XAlB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction + { + for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction + { + a01 = L + j*cs_a +(j+D_NR); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //(row 3):FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + //(Row 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + //(Row 1): FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + + + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j + D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] + } + if(n_remainder == 2) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] + } + if(n_remainder == 1) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //(row 3):FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + //(Row 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + //(Row 1): FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + } + } + if(i<0) + i += D_NR; + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //store(B11[x][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointwr to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + if(n_remainder == 3) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + } + if(n_remainder == 2) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + } + if(n_remainder == 1) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + } + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm16); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm16); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm16); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm16); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + + ///GEMM processing stars/// + + for(k = 0; k < k_iter; k++) + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + + } + + ///GEMM code ends/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -= ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) + } + + } + m_remainder -= 4; + i -= 4; + } + if(m_remainder) + { + dtrsm_small_XAlB_unitDiag(L, B, AlphaVal, m_remainder, n, cs_a, cs_b); + } + return BLIS_SUCCESS; +} + + +/*implements TRSM for the case XA = alpha * B + *A is lower triangular, non-unit diagonal, no transpose + *dimensions: X:mxn A:nxn B: mxn + */ + +/* <---b11 <---a11 + ***************** * + *b01*b11* * * * * + ^ * * * * * ^ * * + | ***************** | ******* + | * * * * * | * * * + | * * * * * a01* * * +b10 ***************** ************* + * * * * * * * * * + * * * * * * * * * + ***************** ******************* + +*/ +static err_t bli_dtrsm_small_XAutB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction + { + for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction + { + a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + ymm7 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm0 = _mm256_div_pd(ymm7, ymm0); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + //extract a33 + ymm7 = _mm256_permute_pd(ymm0, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm11 = _mm256_mul_pd(ymm11, ymm7); + + ymm15 = _mm256_mul_pd(ymm15, ymm7); + + //extract a22 + ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + //(Row 3): FMA operations + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + ymm10 = _mm256_mul_pd(ymm10, ymm7); + + ymm14 = _mm256_mul_pd(ymm14, ymm7); + + //extract a11 + ymm7 = _mm256_permute_pd(ymm0, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + ymm9 = _mm256_mul_pd(ymm9, ymm7); + + ymm13 = _mm256_mul_pd(ymm13, ymm7); + + //extract A00 + ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + ymm8 = _mm256_mul_pd(ymm8, ymm7); + + ymm12 = _mm256_mul_pd(ymm12, ymm7); + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[x][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + + a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] + } + if(n_remainder == 2) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] + } + if(n_remainder == 1) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + ymm7 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm0 = _mm256_div_pd(ymm7, ymm0); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + //extract a33 + ymm7 = _mm256_permute_pd(ymm0, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm11 = _mm256_mul_pd(ymm11, ymm7); + + ymm15 = _mm256_mul_pd(ymm15, ymm7); + + //extract a22 + ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + //(Row 3): FMA operations + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + ymm10 = _mm256_mul_pd(ymm10, ymm7); + + ymm14 = _mm256_mul_pd(ymm14, ymm7); + + //extract a11 + ymm7 = _mm256_permute_pd(ymm0, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + ymm9 = _mm256_mul_pd(ymm9, ymm7); + + ymm13 = _mm256_mul_pd(ymm13, ymm7); + + //extract A00 + ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + ymm8 = _mm256_mul_pd(ymm8, ymm7); + + ymm12 = _mm256_mul_pd(ymm12, ymm7); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + } + } + if(i<0) + i += D_NR; + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction + { + a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR*cs_a; //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a33 + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm3 = _mm256_mul_pd(ymm3, ymm15); + + //extract a22 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + ymm2 = _mm256_mul_pd(ymm2, ymm15); + + //extract a11 + ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + ymm1 = _mm256_mul_pd(ymm1, ymm15); + + //extract A00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + ymm0 = _mm256_mul_pd(ymm0, ymm15); + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) + { + + a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + if(n_remainder == 3) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + } + if(n_remainder == 2) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + } + if(n_remainder == 1) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + } + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a33 + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm3 = _mm256_mul_pd(ymm3, ymm15); + + //extract a22 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + ymm2 = _mm256_mul_pd(ymm2, ymm15); + + //extract a11 + ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + ymm1 = _mm256_mul_pd(ymm1, ymm15); + + //extract A00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + ymm0 = _mm256_mul_pd(ymm0, ymm15); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) + } + + } + m_remainder -= 4; + i -= 4; + } + if(m_remainder) ///implementation for remainder rows + { + dtrsm_small_XAutB(L, B, AlphaVal, m_remainder, n, cs_a, cs_b); + } + return BLIS_SUCCESS; +} + +/*implements TRSM for the case XA = alpha * B + *A is lower triangular, unit-diagonal, no transpose + *dimensions: X:mxn A:nxn B: mxn + */ + +/* <---b11 <---a11 + ***************** * + *b01*b11* * * * * + ^ * * * * * ^ * * + | ***************** | ******* + | * * * * * | * * * + | * * * * * a01* * * +b10 ***************** ************* + * * * * * * * * * + * * * * * * * * * + ***************** ******************* + +*/ +static err_t bli_dtrsm_small_XAutB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction + { + for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction + { + a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + //(Row 3): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + //(Row 3): FMA operations + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + //(ROW 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + //(Row 1):FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[x][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + + a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] + } + if(n_remainder == 2) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] + } + if(n_remainder == 1) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + //(Row 3): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + //(Row 3): FMA operations + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + //(ROW 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + //(Row 1):FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + + } + } + if(i<0) + i += D_NR; + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction + { + a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR*cs_a; //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) + { + + a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + if(n_remainder == 3) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + } + if(n_remainder == 2) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + } + if(n_remainder == 1) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + } + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) + } + + } + m_remainder -= 4; + i -= 4; + } + if(m_remainder) + { + dtrsm_small_XAutB_unitDiag(a->buffer, b->buffer,AlphaVal, m_remainder, n, cs_a, cs_b); + } + return BLIS_SUCCESS; +} + + +/* + * AX = Alpha*B, Single precision, A: lower triangular + * This kernel implementation supports matrices A and B such that m is equal to BLI_AlXB_M_SP and n is mutiple of 8 + */ + +static err_t bli_strsm_small_AlXB ( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + obj_t alpha, beta; // gemm parameters + obj_t Ga, Gb, Gc; // for GEMM + int m = bli_obj_length(b); // number of rows of matrix B + int n = bli_obj_width(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int j; + int blk_size = 8; + int isUnitDiag = bli_obj_has_unit_diag(a); + + float alphaVal; + float *L = a->buffer; + float *B = b->buffer; + + if (m != BLI_AlXB_M_SP || (n&7) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); + + /* Small _GEMM preparation code */ + bli_obj_create( BLIS_FLOAT, 1, 1, 0, 0, &alpha ); + bli_obj_create( BLIS_FLOAT, 1, 1, 0, 0, &beta ); + + /* B = B - A*B */ + bli_setsc( -(1.0), 0.0, &alpha ); + bli_setsc( (1.0), 0.0, &beta ); + + + bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, blk_size, a->buffer, rsa, lda, &Ga); + bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, n, b->buffer, rsb, ldb, &Gb); + bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, n, b->buffer, rsb, ldb, &Gc); + + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Ga ); + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gb ); + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gc ); + + //first block of trsm + Gb.buffer = (void*)(B + i); + + //trsm of first 8xn block + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + blis_strsm_microkernel_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_strsm_microkernel = blis_strsm_microkernel; + } + else + { + blis_strsm_microkernel_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; + } + bli_setsc( alphaVal, 0.0, &beta ); + } + else + { + if (isUnitDiag == 0) + { + blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_strsm_microkernel = blis_strsm_microkernel; + } + else + { + blis_strsm_microkernel_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; + } + } + + //gemm update + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); + + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } + + //trsm of remaining blocks + for (i = blk_size; i < m; i += blk_size) + { + Gb.buffer = (void*)(B + i); + + fp_blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); + + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } + + } // End of for loop - i + + return BLIS_SUCCESS; +} + + + +/* + * XA' = Alpha*B, Single precision, A: lower triangular + * This kernel implementation supports matrices A and B such that + * m and n are multiples of 8 and n is less than or equal to BLI_XAltB_N_SP + */ +static err_t bli_strsm_small_XAltB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int m = bli_obj_length(a); // number of rows of matrix B + int n = bli_obj_length(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int isUnitDiag = bli_obj_has_unit_diag(a); + + float alphaVal; + float *L = a->buffer; + float *B = b->buffer; + + if ((m&7) != 0 || (n&7) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( n > BLI_XAltB_N_SP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); + + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + trsm_XAtB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + } + else + { + if (isUnitDiag == 0) + { + trsm_XAtB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + trsm_XAtB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + } + return BLIS_SUCCESS; +} + +/* + * A'X = Alpha*B, Single precision, A: upper triangular + * This kernel implementation supports matrices A and B such that + * m and n are multiples of 8, m is less than or equal to BLI_AutXB_M_SP and n is less than or equal to BLI_AutXB_N_SP + */ +static err_t bli_strsm_small_AutXB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int m = bli_obj_width(a); // number of rows of matrix A (since At, so width is taken) + int n = bli_obj_width(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int isUnitDiag = bli_obj_has_unit_diag(a); + + float alphaVal; + float *L = a->buffer; + float *B = b->buffer; + + if ((m&7) != 0 || (n&7) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( m > BLI_AutXB_M_SP || n > BLI_AutXB_N_SP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); + + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + trsm_AutXB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + } + else + { + if (isUnitDiag == 0) + { + trsm_AutXB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + trsm_AutXB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + } + return BLIS_SUCCESS; +} + +///////////////////////////// AX=B /////////////////////////////// +static void blis_strsm_microkernel_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) +{ + float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags; + __m256 alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols +} + +static void blis_strsm_microkernel_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) +{ + //float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags; + __m256 alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //8th col + //ptr_l += cs_l; + //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols +} + +static void blis_strsm_microkernel_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //8th col + //ptr_l += cs_l; + //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols +} + +static void blis_strsm_microkernel(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols +} +static void blis_dtrsm_microkernel_alpha(double *ptr_l, + double *ptr_b, + int m, + int n, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ) +{ + int j; + int n_remainder = n%4; + int cs_b_offset[2]; + double *ptr_b_dup; + double ones = 1.0; + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags; + __m256d alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + + reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); + alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); + + //if(m % 4 == 0) + //{ + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); + reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); + + for(j = 0;(j+3) < n; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + } + ptr_b_dup = ptr_b; + if(n_remainder == 3) + { + + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_broadcast_sd((double const *)&ones); + mat_b_col[3] = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_broadcast_sd((double const *)&ones); + mat_b_col[2] = _mm256_broadcast_sd((double const *)&ones); + mat_b_col[3] = _mm256_broadcast_sd((double const *)&ones); + } + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + //Store the computed B columns + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + } + + //} + +} + +#if OPT_CACHE_BLOCKING_L1 //new intrinsic kernels +static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0) + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0) + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done + +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} +#else //rel 1.0 intrisic kernels (NOT OPT_CACHE_BLOCKING_L1) +static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + __m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + __m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); + mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); + mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); + mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); + mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); + mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); + mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); + mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); + mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); + + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + //__m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0) + mat_b_col[0] = mat_b_rearr[0][0]; + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): already done + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + //__m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); + mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); + mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); + mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); + mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); + + //(Row0) + mat_b_col[0] = mat_b_rearr[0][0]; + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); + mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); + mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); + mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); + mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); + + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): already done + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} +#endif //OPT_CACHE_BLOCKING_L1 + +//////////////////////////// AutX=B /////////////////////// +static void trsm_AutXB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += 8; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ +#endif + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_AutXB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += 8; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_AutXB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int i, i1, i2, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + + //(Row0) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ +#endif + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done + +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + //float ones = 1.0; + int i, i1, i2, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done + +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} +#endif diff --git a/kernels/zen2/.gitignore b/kernels/zen2/.gitignore new file mode 100644 index 000000000..5e7d2734c --- /dev/null +++ b/kernels/zen2/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/ref_kernels/bli_cntx_ref.c b/ref_kernels/bli_cntx_ref.c index ea2a3554d..36cdd52dc 100644 --- a/ref_kernels/bli_cntx_ref.c +++ b/ref_kernels/bli_cntx_ref.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/blx_gemm_front.c b/sandbox/ref99/blx_gemm_front.c index bb6ba4a8d..399f750a5 100644 --- a/sandbox/ref99/blx_gemm_front.c +++ b/sandbox/ref99/blx_gemm_front.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/blx_gemm_int.c b/sandbox/ref99/blx_gemm_int.c index c807fc76e..525f72d5d 100644 --- a/sandbox/ref99/blx_gemm_int.c +++ b/sandbox/ref99/blx_gemm_int.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/cntl/blx_gemm_cntl.c b/sandbox/ref99/cntl/blx_gemm_cntl.c index 9859e85a2..33c97716a 100644 --- a/sandbox/ref99/cntl/blx_gemm_cntl.c +++ b/sandbox/ref99/cntl/blx_gemm_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/oapi/bli_gemmnat.c b/sandbox/ref99/oapi/bli_gemmnat.c index 865c7cff4..e9246338d 100644 --- a/sandbox/ref99/oapi/bli_gemmnat.c +++ b/sandbox/ref99/oapi/bli_gemmnat.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/vars/blx_gemm_blk_var1.c b/sandbox/ref99/vars/blx_gemm_blk_var1.c index ef8c07b1d..dc41b97ff 100644 --- a/sandbox/ref99/vars/blx_gemm_blk_var1.c +++ b/sandbox/ref99/vars/blx_gemm_blk_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/vars/blx_gemm_blk_var2.c b/sandbox/ref99/vars/blx_gemm_blk_var2.c index f272952b0..d7d128c35 100644 --- a/sandbox/ref99/vars/blx_gemm_blk_var2.c +++ b/sandbox/ref99/vars/blx_gemm_blk_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/vars/blx_gemm_ker_var2.c b/sandbox/ref99/vars/blx_gemm_ker_var2.c index 61842411a..10c6b81ad 100644 --- a/sandbox/ref99/vars/blx_gemm_ker_var2.c +++ b/sandbox/ref99/vars/blx_gemm_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/vars/blx_gemm_var.h b/sandbox/ref99/vars/blx_gemm_var.h index 32b975d1a..a2a3de9bb 100644 --- a/sandbox/ref99/vars/blx_gemm_var.h +++ b/sandbox/ref99/vars/blx_gemm_var.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/vars/other/blx_gemm_ker_var2rr.c b/sandbox/ref99/vars/other/blx_gemm_ker_var2rr.c index 8a5c1d156..7cbd402e0 100644 --- a/sandbox/ref99/vars/other/blx_gemm_ker_var2rr.c +++ b/sandbox/ref99/vars/other/blx_gemm_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/vars/other/blx_gemm_ker_var2sl.c b/sandbox/ref99/vars/other/blx_gemm_ker_var2sl.c index 4b0523e37..2d46886b7 100644 --- a/sandbox/ref99/vars/other/blx_gemm_ker_var2sl.c +++ b/sandbox/ref99/vars/other/blx_gemm_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/3/test_herk.c b/test/3/test_herk.c index b963f944b..ebb0bd8d2 100644 --- a/test/3/test_herk.c +++ b/test/3/test_herk.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/3/test_trmm.c b/test/3/test_trmm.c index 2fa7fe52d..08f202992 100644 --- a/test/3/test_trmm.c +++ b/test/3/test_trmm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/3/test_trsm.c b/test/3/test_trsm.c index 2e5ff0a53..8b7a9cb1d 100644 --- a/test/3/test_trsm.c +++ b/test/3/test_trsm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/Makefile b/test/Makefile index 0166543c4..732ef0dd0 100644 --- a/test/Makefile +++ b/test/Makefile @@ -5,7 +5,7 @@ # libraries. # # Copyright (C) 2014, The University of Texas at Austin -# Copyright (C) 2017, Advanced Micro Devices, Inc. +# Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are diff --git a/test/test_axpyv.c b/test/test_axpyv.c index 54a4f6134..d3e91ef48 100644 --- a/test/test_axpyv.c +++ b/test/test_axpyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/test_dotv.c b/test/test_dotv.c index d5bebea5a..5f4aeac39 100644 --- a/test/test_dotv.c +++ b/test/test_dotv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/thread_ranges/test_ranges.c b/test/thread_ranges/test_ranges.c index 9bf293ca5..dbbe7f2d8 100644 --- a/test/thread_ranges/test_ranges.c +++ b/test/thread_ranges/test_ranges.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_addm.c b/testsuite/src/test_addm.c index 821c8b55e..f7c21b733 100644 --- a/testsuite/src/test_addm.c +++ b/testsuite/src/test_addm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_addm.h b/testsuite/src/test_addm.h index 815f5db85..0dbdbfa2e 100644 --- a/testsuite/src/test_addm.h +++ b/testsuite/src/test_addm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_addv.c b/testsuite/src/test_addv.c index c394ea1d8..9e216ab4d 100644 --- a/testsuite/src/test_addv.c +++ b/testsuite/src/test_addv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_addv.h b/testsuite/src/test_addv.h index 1b9982e31..eba5a9220 100644 --- a/testsuite/src/test_addv.h +++ b/testsuite/src/test_addv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_amaxv.c b/testsuite/src/test_amaxv.c index 72bdb2cc9..ed2a821fe 100644 --- a/testsuite/src/test_amaxv.c +++ b/testsuite/src/test_amaxv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_amaxv.h b/testsuite/src/test_amaxv.h index 4c382593f..46d87b37f 100644 --- a/testsuite/src/test_amaxv.h +++ b/testsuite/src/test_amaxv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpbyv.c b/testsuite/src/test_axpbyv.c index 8def7b32d..a82ff6e25 100644 --- a/testsuite/src/test_axpbyv.c +++ b/testsuite/src/test_axpbyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpbyv.h b/testsuite/src/test_axpbyv.h index a8fcd2dfa..9b318dba1 100644 --- a/testsuite/src/test_axpbyv.h +++ b/testsuite/src/test_axpbyv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpy2v.c b/testsuite/src/test_axpy2v.c index c23443ab0..eeebf15e7 100644 --- a/testsuite/src/test_axpy2v.c +++ b/testsuite/src/test_axpy2v.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpy2v.h b/testsuite/src/test_axpy2v.h index dc465792d..c695a643b 100644 --- a/testsuite/src/test_axpy2v.h +++ b/testsuite/src/test_axpy2v.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpyf.c b/testsuite/src/test_axpyf.c index 155e442b0..7a85b2212 100644 --- a/testsuite/src/test_axpyf.c +++ b/testsuite/src/test_axpyf.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpyf.h b/testsuite/src/test_axpyf.h index 179acb9bb..9dd1dadc2 100644 --- a/testsuite/src/test_axpyf.h +++ b/testsuite/src/test_axpyf.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpym.c b/testsuite/src/test_axpym.c index 0138d822a..222fda33d 100644 --- a/testsuite/src/test_axpym.c +++ b/testsuite/src/test_axpym.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpym.h b/testsuite/src/test_axpym.h index 29819640f..632720284 100644 --- a/testsuite/src/test_axpym.h +++ b/testsuite/src/test_axpym.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpyv.c b/testsuite/src/test_axpyv.c index 89b505f4c..81d4f3770 100644 --- a/testsuite/src/test_axpyv.c +++ b/testsuite/src/test_axpyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpyv.h b/testsuite/src/test_axpyv.h index a5ce3ea03..c96a9096b 100644 --- a/testsuite/src/test_axpyv.h +++ b/testsuite/src/test_axpyv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_copym.c b/testsuite/src/test_copym.c index 2532a50c7..1aab1d287 100644 --- a/testsuite/src/test_copym.c +++ b/testsuite/src/test_copym.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_copym.h b/testsuite/src/test_copym.h index 2a876ea68..560de0e9a 100644 --- a/testsuite/src/test_copym.h +++ b/testsuite/src/test_copym.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_copyv.c b/testsuite/src/test_copyv.c index 70cb80f23..4350e95ee 100644 --- a/testsuite/src/test_copyv.c +++ b/testsuite/src/test_copyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_copyv.h b/testsuite/src/test_copyv.h index 1d413f75b..2beb3212d 100644 --- a/testsuite/src/test_copyv.h +++ b/testsuite/src/test_copyv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotaxpyv.c b/testsuite/src/test_dotaxpyv.c index b9b8d99c3..391c119bb 100644 --- a/testsuite/src/test_dotaxpyv.c +++ b/testsuite/src/test_dotaxpyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotaxpyv.h b/testsuite/src/test_dotaxpyv.h index 742133970..ce82227f4 100644 --- a/testsuite/src/test_dotaxpyv.h +++ b/testsuite/src/test_dotaxpyv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotv.c b/testsuite/src/test_dotv.c index 0b0404af3..347ce9e62 100644 --- a/testsuite/src/test_dotv.c +++ b/testsuite/src/test_dotv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotv.h b/testsuite/src/test_dotv.h index d1b3b0e29..2f000128b 100644 --- a/testsuite/src/test_dotv.h +++ b/testsuite/src/test_dotv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotxaxpyf.c b/testsuite/src/test_dotxaxpyf.c index 80638d110..c73ab6c9d 100644 --- a/testsuite/src/test_dotxaxpyf.c +++ b/testsuite/src/test_dotxaxpyf.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotxaxpyf.h b/testsuite/src/test_dotxaxpyf.h index 72b93a637..6bfcd2655 100644 --- a/testsuite/src/test_dotxaxpyf.h +++ b/testsuite/src/test_dotxaxpyf.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotxf.c b/testsuite/src/test_dotxf.c index cac443ac6..8a1eca4eb 100644 --- a/testsuite/src/test_dotxf.c +++ b/testsuite/src/test_dotxf.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotxf.h b/testsuite/src/test_dotxf.h index 8940e6a75..06cac584e 100644 --- a/testsuite/src/test_dotxf.h +++ b/testsuite/src/test_dotxf.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotxv.c b/testsuite/src/test_dotxv.c index 64ab90e02..da42e6ae4 100644 --- a/testsuite/src/test_dotxv.c +++ b/testsuite/src/test_dotxv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotxv.h b/testsuite/src/test_dotxv.h index 02009b5a9..a3e2ca48f 100644 --- a/testsuite/src/test_dotxv.h +++ b/testsuite/src/test_dotxv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemm.c b/testsuite/src/test_gemm.c index 6dae4301e..e941946e1 100644 --- a/testsuite/src/test_gemm.c +++ b/testsuite/src/test_gemm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemm.h b/testsuite/src/test_gemm.h index f1c41bb95..78364bc24 100644 --- a/testsuite/src/test_gemm.h +++ b/testsuite/src/test_gemm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemm_ukr.c b/testsuite/src/test_gemm_ukr.c index 2017c70dc..66e84d644 100644 --- a/testsuite/src/test_gemm_ukr.c +++ b/testsuite/src/test_gemm_ukr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemm_ukr.h b/testsuite/src/test_gemm_ukr.h index d20c47c62..cd09ef3f6 100644 --- a/testsuite/src/test_gemm_ukr.h +++ b/testsuite/src/test_gemm_ukr.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemmtrsm_ukr.c b/testsuite/src/test_gemmtrsm_ukr.c index 20ceac1c6..e2cf10ab3 100644 --- a/testsuite/src/test_gemmtrsm_ukr.c +++ b/testsuite/src/test_gemmtrsm_ukr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemmtrsm_ukr.h b/testsuite/src/test_gemmtrsm_ukr.h index 8bf52c4eb..5fd3cc0ba 100644 --- a/testsuite/src/test_gemmtrsm_ukr.h +++ b/testsuite/src/test_gemmtrsm_ukr.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemv.c b/testsuite/src/test_gemv.c index 022fd2b56..e6090e1c5 100644 --- a/testsuite/src/test_gemv.c +++ b/testsuite/src/test_gemv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemv.h b/testsuite/src/test_gemv.h index 2de09095e..8e7284486 100644 --- a/testsuite/src/test_gemv.h +++ b/testsuite/src/test_gemv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_ger.c b/testsuite/src/test_ger.c index 672077ec1..b44fe6ba6 100644 --- a/testsuite/src/test_ger.c +++ b/testsuite/src/test_ger.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_ger.h b/testsuite/src/test_ger.h index f053a73b9..5b75babe6 100644 --- a/testsuite/src/test_ger.h +++ b/testsuite/src/test_ger.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_hemm.c b/testsuite/src/test_hemm.c index 12afa3698..0145dd0df 100644 --- a/testsuite/src/test_hemm.c +++ b/testsuite/src/test_hemm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_hemm.h b/testsuite/src/test_hemm.h index f3ff7b90f..7db76afa1 100644 --- a/testsuite/src/test_hemm.h +++ b/testsuite/src/test_hemm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_hemv.c b/testsuite/src/test_hemv.c index 0976e0bc8..02e205392 100644 --- a/testsuite/src/test_hemv.c +++ b/testsuite/src/test_hemv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_hemv.h b/testsuite/src/test_hemv.h index 701a79030..e522690d1 100644 --- a/testsuite/src/test_hemv.h +++ b/testsuite/src/test_hemv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_her.c b/testsuite/src/test_her.c index c545e00a7..c122f6ce5 100644 --- a/testsuite/src/test_her.c +++ b/testsuite/src/test_her.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_her.h b/testsuite/src/test_her.h index fed1e45ef..a6aaa55b4 100644 --- a/testsuite/src/test_her.h +++ b/testsuite/src/test_her.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_her2.c b/testsuite/src/test_her2.c index d6a8686ad..1ed6b3bb9 100644 --- a/testsuite/src/test_her2.c +++ b/testsuite/src/test_her2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_her2.h b/testsuite/src/test_her2.h index 6e4d26a64..c2711cfb1 100644 --- a/testsuite/src/test_her2.h +++ b/testsuite/src/test_her2.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_her2k.c b/testsuite/src/test_her2k.c index 9aed2d968..0158e25a2 100644 --- a/testsuite/src/test_her2k.c +++ b/testsuite/src/test_her2k.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_her2k.h b/testsuite/src/test_her2k.h index 36382ed9c..a481dac72 100644 --- a/testsuite/src/test_her2k.h +++ b/testsuite/src/test_her2k.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_herk.c b/testsuite/src/test_herk.c index eda56d2aa..abe4e70b1 100644 --- a/testsuite/src/test_herk.c +++ b/testsuite/src/test_herk.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_herk.h b/testsuite/src/test_herk.h index 6235cb761..1702bd8b9 100644 --- a/testsuite/src/test_herk.h +++ b/testsuite/src/test_herk.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_libblis.c b/testsuite/src/test_libblis.c index 040aea12d..5c2dd2ed9 100644 --- a/testsuite/src/test_libblis.c +++ b/testsuite/src/test_libblis.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_libblis.h b/testsuite/src/test_libblis.h index 4a3dd0ffc..ad1ee3c2c 100644 --- a/testsuite/src/test_libblis.h +++ b/testsuite/src/test_libblis.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_normfm.c b/testsuite/src/test_normfm.c index 7d80f7c17..c4b9a0105 100644 --- a/testsuite/src/test_normfm.c +++ b/testsuite/src/test_normfm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_normfm.h b/testsuite/src/test_normfm.h index b79f6b7bb..a24b5e5ba 100644 --- a/testsuite/src/test_normfm.h +++ b/testsuite/src/test_normfm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_normfv.c b/testsuite/src/test_normfv.c index 83210f168..3bcce35af 100644 --- a/testsuite/src/test_normfv.c +++ b/testsuite/src/test_normfv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_normfv.h b/testsuite/src/test_normfv.h index 2193c43ee..afa535006 100644 --- a/testsuite/src/test_normfv.h +++ b/testsuite/src/test_normfv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_randm.c b/testsuite/src/test_randm.c index f5ef20629..223007dba 100644 --- a/testsuite/src/test_randm.c +++ b/testsuite/src/test_randm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_randm.h b/testsuite/src/test_randm.h index 9c8c87886..e44464962 100644 --- a/testsuite/src/test_randm.h +++ b/testsuite/src/test_randm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_randv.c b/testsuite/src/test_randv.c index 03f98a9b9..951c8c3ec 100644 --- a/testsuite/src/test_randv.c +++ b/testsuite/src/test_randv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_randv.h b/testsuite/src/test_randv.h index 3574c19e5..bb658dfd7 100644 --- a/testsuite/src/test_randv.h +++ b/testsuite/src/test_randv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scal2m.c b/testsuite/src/test_scal2m.c index 7ed1ec49b..e8440fc46 100644 --- a/testsuite/src/test_scal2m.c +++ b/testsuite/src/test_scal2m.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scal2m.h b/testsuite/src/test_scal2m.h index 3abcd9a14..262723f4e 100644 --- a/testsuite/src/test_scal2m.h +++ b/testsuite/src/test_scal2m.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scal2v.c b/testsuite/src/test_scal2v.c index b5b2a3d65..c200e13fc 100644 --- a/testsuite/src/test_scal2v.c +++ b/testsuite/src/test_scal2v.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scal2v.h b/testsuite/src/test_scal2v.h index 3ab6b3c42..75b5cfe4a 100644 --- a/testsuite/src/test_scal2v.h +++ b/testsuite/src/test_scal2v.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scalm.c b/testsuite/src/test_scalm.c index 284e23ab6..6219c71df 100644 --- a/testsuite/src/test_scalm.c +++ b/testsuite/src/test_scalm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scalm.h b/testsuite/src/test_scalm.h index 1723f51dc..3b98617b2 100644 --- a/testsuite/src/test_scalm.h +++ b/testsuite/src/test_scalm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scalv.c b/testsuite/src/test_scalv.c index 61b3f5fbe..142b5e410 100644 --- a/testsuite/src/test_scalv.c +++ b/testsuite/src/test_scalv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scalv.h b/testsuite/src/test_scalv.h index 9092ae359..144b41675 100644 --- a/testsuite/src/test_scalv.h +++ b/testsuite/src/test_scalv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_setm.c b/testsuite/src/test_setm.c index 630ced831..80cebd64e 100644 --- a/testsuite/src/test_setm.c +++ b/testsuite/src/test_setm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_setm.h b/testsuite/src/test_setm.h index f5cd32aa1..027184031 100644 --- a/testsuite/src/test_setm.h +++ b/testsuite/src/test_setm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_setv.c b/testsuite/src/test_setv.c index a0ed3ee97..10f0348c7 100644 --- a/testsuite/src/test_setv.c +++ b/testsuite/src/test_setv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_setv.h b/testsuite/src/test_setv.h index b2494a17c..4e02d489e 100644 --- a/testsuite/src/test_setv.h +++ b/testsuite/src/test_setv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_subm.c b/testsuite/src/test_subm.c index 8c6a83831..63b48eedc 100644 --- a/testsuite/src/test_subm.c +++ b/testsuite/src/test_subm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_subm.h b/testsuite/src/test_subm.h index c7e7e93ce..e39eff828 100644 --- a/testsuite/src/test_subm.h +++ b/testsuite/src/test_subm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_subv.c b/testsuite/src/test_subv.c index 7d7a107dd..3a48f02a4 100644 --- a/testsuite/src/test_subv.c +++ b/testsuite/src/test_subv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_subv.h b/testsuite/src/test_subv.h index 30fd8bba8..5dbe46589 100644 --- a/testsuite/src/test_subv.h +++ b/testsuite/src/test_subv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_symm.c b/testsuite/src/test_symm.c index e36147251..2ac7b4106 100644 --- a/testsuite/src/test_symm.c +++ b/testsuite/src/test_symm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_symm.h b/testsuite/src/test_symm.h index fe960d4fe..bf50bf65d 100644 --- a/testsuite/src/test_symm.h +++ b/testsuite/src/test_symm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_symv.c b/testsuite/src/test_symv.c index a1f914142..5ae5f30be 100644 --- a/testsuite/src/test_symv.c +++ b/testsuite/src/test_symv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_symv.h b/testsuite/src/test_symv.h index 0a0a833c5..5dba0624c 100644 --- a/testsuite/src/test_symv.h +++ b/testsuite/src/test_symv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syr.c b/testsuite/src/test_syr.c index f328d061b..69376b970 100644 --- a/testsuite/src/test_syr.c +++ b/testsuite/src/test_syr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syr.h b/testsuite/src/test_syr.h index d616f969b..455e18ff1 100644 --- a/testsuite/src/test_syr.h +++ b/testsuite/src/test_syr.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syr2.c b/testsuite/src/test_syr2.c index e79bfeca6..42d65c00e 100644 --- a/testsuite/src/test_syr2.c +++ b/testsuite/src/test_syr2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syr2.h b/testsuite/src/test_syr2.h index 6f0998354..d6c1f3c10 100644 --- a/testsuite/src/test_syr2.h +++ b/testsuite/src/test_syr2.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syr2k.c b/testsuite/src/test_syr2k.c index e1346692d..4d83bb88c 100644 --- a/testsuite/src/test_syr2k.c +++ b/testsuite/src/test_syr2k.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syr2k.h b/testsuite/src/test_syr2k.h index 9ceb3befd..edf893c29 100644 --- a/testsuite/src/test_syr2k.h +++ b/testsuite/src/test_syr2k.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syrk.c b/testsuite/src/test_syrk.c index d6ca4b3bd..65d978bb0 100644 --- a/testsuite/src/test_syrk.c +++ b/testsuite/src/test_syrk.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syrk.h b/testsuite/src/test_syrk.h index e0d461c10..8cad72456 100644 --- a/testsuite/src/test_syrk.h +++ b/testsuite/src/test_syrk.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trmm.c b/testsuite/src/test_trmm.c index be6bb941e..a1decd37c 100644 --- a/testsuite/src/test_trmm.c +++ b/testsuite/src/test_trmm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trmm.h b/testsuite/src/test_trmm.h index 0c0b617dd..a84ca1d29 100644 --- a/testsuite/src/test_trmm.h +++ b/testsuite/src/test_trmm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trmm3.c b/testsuite/src/test_trmm3.c index ba9431a0b..17ba2190b 100644 --- a/testsuite/src/test_trmm3.c +++ b/testsuite/src/test_trmm3.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trmm3.h b/testsuite/src/test_trmm3.h index 6150b7023..ee9490036 100644 --- a/testsuite/src/test_trmm3.h +++ b/testsuite/src/test_trmm3.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trmv.c b/testsuite/src/test_trmv.c index b4b2f386d..71acc90ba 100644 --- a/testsuite/src/test_trmv.c +++ b/testsuite/src/test_trmv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trmv.h b/testsuite/src/test_trmv.h index 185aeb8a3..1fae8331f 100644 --- a/testsuite/src/test_trmv.h +++ b/testsuite/src/test_trmv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trsm.c b/testsuite/src/test_trsm.c index fa50bf790..fa0d8e7c3 100644 --- a/testsuite/src/test_trsm.c +++ b/testsuite/src/test_trsm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2019, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trsm.h b/testsuite/src/test_trsm.h index 2738511f2..ee23b2c7a 100644 --- a/testsuite/src/test_trsm.h +++ b/testsuite/src/test_trsm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trsm_ukr.c b/testsuite/src/test_trsm_ukr.c index 7d3df41c9..ee468dbd3 100644 --- a/testsuite/src/test_trsm_ukr.c +++ b/testsuite/src/test_trsm_ukr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trsm_ukr.h b/testsuite/src/test_trsm_ukr.h index 71685bb2a..22c667636 100644 --- a/testsuite/src/test_trsm_ukr.h +++ b/testsuite/src/test_trsm_ukr.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trsv.c b/testsuite/src/test_trsv.c index b05f7ab97..12543cd9a 100644 --- a/testsuite/src/test_trsv.c +++ b/testsuite/src/test_trsv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trsv.h b/testsuite/src/test_trsv.h index b2e85469d..5f5fa4eb0 100644 --- a/testsuite/src/test_trsv.h +++ b/testsuite/src/test_trsv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_xpbyv.c b/testsuite/src/test_xpbyv.c index 75ad98f6f..197de86e7 100644 --- a/testsuite/src/test_xpbyv.c +++ b/testsuite/src/test_xpbyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_xpbyv.h b/testsuite/src/test_xpbyv.h index 3b2e7bee2..16eb77216 100644 --- a/testsuite/src/test_xpbyv.h +++ b/testsuite/src/test_xpbyv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are