From 825244052d95d165e261808dba6712e97d17a41b Mon Sep 17 00:00:00 2001 From: "Rayan, Rohan" Date: Wed, 1 Jul 2026 14:57:59 +0530 Subject: [PATCH] SGEMM Zen5 RD kernel optimization Removed function calls to 6x48, 6x32, 6x16, 6x8, 6x4 fringe kernels. These are now handled in the main kernel directly Introduced new fringe kernels 6x3, 6x2 in the n-direction Removed fringe kernel calls to AVX2 code path Access pattern inside the kernel changed so that the C-matrix stores are along the cache line Various instruction level optimizations These kerels are enabled also on the Zen6 code path Tiny path modified to take these newer kernels on zen5 and zen6 Added Gtests for these new kernels AMD-Internal: CPUPL-8424, SWLCSG-3272 Co-authored-by: Rohan Rayan rohrayan@amd.com --- config/zen5/bli_cntx_init_zen5.c | 4 +- config/zen5/bli_config_zen5.h | 2 +- config/zen6/bli_cntx_init_zen6.c | 4 +- config/zen6/bli_config_zen6.h | 2 +- .../inc/common/blis_version_defs.h | 2 + .../ukr/gemm/sgemm/sgemm_ukernel.cpp | 130 + kernels/zen5/3/bli_gemm_tiny_zen5.c | 101 + kernels/zen5/3/bli_gemm_tiny_zen5.h | 59 + .../3/sup/bli_gemmsup_rd_zen5_asm_s6x64.c | 5202 +++++++++++++++++ .../3/sup/bli_gemmsup_rd_zen5_asm_s6x64.h | 700 +++ .../3/sup/bli_gemmsup_rd_zen5_asm_s6x64m.c | 1712 ++++++ .../3/sup/bli_gemmsup_rd_zen5_asm_s6x64n.c | 2456 ++++++++ kernels/zen5/bli_kernels_zen5.h | 30 +- 13 files changed, 10397 insertions(+), 7 deletions(-) create mode 100644 kernels/zen5/3/bli_gemm_tiny_zen5.c create mode 100644 kernels/zen5/3/bli_gemm_tiny_zen5.h create mode 100644 kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.c create mode 100644 kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.h create mode 100644 kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64m.c create mode 100644 kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64n.c diff --git a/config/zen5/bli_cntx_init_zen5.c b/config/zen5/bli_cntx_init_zen5.c index 1e5a009ea..fdf730024 100644 --- a/config/zen5/bli_cntx_init_zen5.c +++ b/config/zen5/bli_cntx_init_zen5.c @@ -314,11 +314,11 @@ void bli_cntx_init_zen5( cntx_t* cntx ) BLIS_CCC, BLIS_DOUBLE, bli_dgemmsup_cv_zen5_asm_24x8m, FALSE, BLIS_RRR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64m, TRUE, - BLIS_RRC, BLIS_FLOAT, bli_sgemmsup_rd_zen4_asm_6x64m, TRUE, + BLIS_RRC, BLIS_FLOAT, bli_sgemmsup_rd_zen5_asm_6x64m, TRUE, BLIS_RCR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64m, TRUE, BLIS_RCC, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64n, TRUE, BLIS_CRR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64m, TRUE, - BLIS_CRC, BLIS_FLOAT, bli_sgemmsup_rd_zen4_asm_6x64n, TRUE, + BLIS_CRC, BLIS_FLOAT, bli_sgemmsup_rd_zen5_asm_6x64n, TRUE, BLIS_CCR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64n, TRUE, BLIS_CCC, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64n, TRUE, diff --git a/config/zen5/bli_config_zen5.h b/config/zen5/bli_config_zen5.h index bf0f91671..91f046ea6 100644 --- a/config/zen5/bli_config_zen5.h +++ b/config/zen5/bli_config_zen5.h @@ -171,7 +171,7 @@ if ( PASTECH2( THRESH_GEMM_, ch, _TINY_ZEN5_AVX2 )( transa, transb, m, n, k, is_parallel ) ) \ LOOKUP_AVX2_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) \ else if ( PASTECH2( THRESH_GEMM_, ch, _TINY_ZEN5_AVX512 )( stor_id, transa, transb, m, n, k, is_parallel ) ) \ - LOOKUP_AVX512_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) \ + LOOKUP_ZEN5_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) \ break; #endif diff --git a/config/zen6/bli_cntx_init_zen6.c b/config/zen6/bli_cntx_init_zen6.c index d7a3d36e6..341c6c6bf 100644 --- a/config/zen6/bli_cntx_init_zen6.c +++ b/config/zen6/bli_cntx_init_zen6.c @@ -314,11 +314,11 @@ void bli_cntx_init_zen6( cntx_t* cntx ) BLIS_CCC, BLIS_DOUBLE, bli_dgemmsup_cv_zen5_asm_24x8m, FALSE, BLIS_RRR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64m, TRUE, - BLIS_RRC, BLIS_FLOAT, bli_sgemmsup_rd_zen4_asm_6x64m, TRUE, + BLIS_RRC, BLIS_FLOAT, bli_sgemmsup_rd_zen5_asm_6x64m, TRUE, BLIS_RCR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64m, TRUE, BLIS_RCC, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64n, TRUE, BLIS_CRR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64m, TRUE, - BLIS_CRC, BLIS_FLOAT, bli_sgemmsup_rd_zen4_asm_6x64n, TRUE, + BLIS_CRC, BLIS_FLOAT, bli_sgemmsup_rd_zen5_asm_6x64n, TRUE, BLIS_CCR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64n, TRUE, BLIS_CCC, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64n, TRUE, diff --git a/config/zen6/bli_config_zen6.h b/config/zen6/bli_config_zen6.h index c85ead46d..c46f79665 100644 --- a/config/zen6/bli_config_zen6.h +++ b/config/zen6/bli_config_zen6.h @@ -171,7 +171,7 @@ if ( PASTECH2( THRESH_GEMM_, ch, _TINY_ZEN6_AVX2 )( transa, transb, m, n, k, is_parallel ) ) \ LOOKUP_AVX2_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) \ else if ( PASTECH2( THRESH_GEMM_, ch, _TINY_ZEN6_AVX512 )( stor_id, transa, transb, m, n, k, is_parallel ) ) \ - LOOKUP_AVX512_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) \ + LOOKUP_ZEN5_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) /* Using the zen5 configuration */ \ break; #endif diff --git a/gtestsuite/testinghelpers/inc/common/blis_version_defs.h b/gtestsuite/testinghelpers/inc/common/blis_version_defs.h index 68225c28c..e15b1a5b2 100644 --- a/gtestsuite/testinghelpers/inc/common/blis_version_defs.h +++ b/gtestsuite/testinghelpers/inc/common/blis_version_defs.h @@ -125,6 +125,8 @@ #define K_bli_dnorm2fv_zen_int_unb_var1 bli_dnorm2fv_zen_int_unb_var1 #define K_bli_sgemmsup_rd_zen4_asm_6x64n bli_sgemmsup_rd_zen4_asm_6x64n #define K_bli_sgemmsup_rd_zen4_asm_6x64m bli_sgemmsup_rd_zen4_asm_6x64m + #define K_bli_sgemmsup_rd_zen5_asm_6x64n bli_sgemmsup_rd_zen5_asm_6x64n + #define K_bli_sgemmsup_rd_zen5_asm_6x64m bli_sgemmsup_rd_zen5_asm_6x64m #define K_bli_sgemmsup_rv_zen4_asm_6x64n bli_sgemmsup_rv_zen4_asm_6x64n #define K_bli_sgemmsup_rv_zen4_asm_6x64m bli_sgemmsup_rv_zen4_asm_6x64m #define K_bli_sgemmsup_rv_zen4_asm_6x64n bli_sgemmsup_rv_zen4_asm_6x64n diff --git a/gtestsuite/testsuite/ukr/gemm/sgemm/sgemm_ukernel.cpp b/gtestsuite/testsuite/ukr/gemm/sgemm/sgemm_ukernel.cpp index 9f869ab00..daadca44c 100644 --- a/gtestsuite/testsuite/ukr/gemm/sgemm/sgemm_ukernel.cpp +++ b/gtestsuite/testsuite/ukr/gemm/sgemm/sgemm_ukernel.cpp @@ -519,6 +519,136 @@ INSTANTIATE_TEST_SUITE_P( #endif // defined(BLIS_KERNELS_ZEN4) && defined(GTEST_AVX512) + +#if defined(BLIS_KERNELS_ZEN5) && defined(GTEST_AVX512) + +/* + The bli_sgemmsup_rd_zen5_asm_6x64m(standalone), accepts inputs with the + following contingency for n. + n <= NR, where NR is 64 + The code structure for the sgemm_sup rd kernels(m-var) are as follows: + In m direction : + Main kernel : Blocks of 6(L6_M) + Fringe kernels : 5 ... 1(L5_M ... L1_M) + In k direction : + Main loop : Blocks of 64(L64_K) + Fringe loop : Blocks of 32, 16, followed by a masked tail for 1..15 + In n direction : + Main kernel : NR = 64(L64_N) + Fringe kernels : With n being 3, 2, 1(AVX512 kernels)(L3_N, L2_N, L1_N) + + The inherent storage scheme format for the kernel is RRC, for C, A and B. + The testing interface allows for testing row-storage(inherent) and col-storage(operation transpose) + of C. We still need to pass the right transpose value pair for A and B, as per the kernel requirement. +*/ + +// Checking with row storage of C +#ifdef K_bli_sgemmsup_rd_zen5_asm_6x64m +INSTANTIATE_TEST_SUITE_P( + bli_sgemmsup_rd_zen5_asm_6x64m_row_stored_c, + sgemmGenericSUP, + ::testing::Combine( + ::testing::Range(gtint_t(1), gtint_t(7), gtint_t(1)), // values of m(L6_M to L1_M) + ::testing::Values(gtint_t(64), // values of n, L64_N + gtint_t(48), // L48_N + gtint_t(32), // L32_N + gtint_t(8), // L8_N + gtint_t(7), // L4_N + L3_N + gtint_t(3), // L3_N + gtint_t(2), // L2_N + gtint_t(1), // L1_N + gtint_t(63)), // Combination of fringe cases for N + ::testing::Values(gtint_t(64), // values of k, L64_K + gtint_t(32), // L32_K + gtint_t(16), // L16_K + gtint_t(12), // L_gt_8_K + gtint_t(7), // L_lt_8_K + gtint_t(256), // 4 * L64_K + gtint_t(303)), // Combination of main and fringe cases for K + ::testing::Values(2.0, 1.0, -1.0), // alpha value + ::testing::Values(1.0, 0.0, -1.0, 2.3), // beta value + ::testing::Values('r'), // storage of c + ::testing::Values(K_bli_sgemmsup_rd_zen5_asm_6x64m), // sgemm_sup_kernel + ::testing::Values(gtint_t(6)), // Micro kernel block MR + ::testing::Values('n'), // transa, has to be N for row storage + ::testing::Values('t'), // transb, has to be T for row storage + ::testing::Values(true), // kernel pref + ::testing::Values(true, false) // memory test + ), + ::sgemmGenericSUPPrint() + ); +#endif + +// Checking with col storage of C +// NOTE : Since we are inducing transpose at operation level, for code coverage, we +// have to interchange m and n instantiations +#ifdef K_bli_sgemmsup_rd_zen5_asm_6x64m +INSTANTIATE_TEST_SUITE_P( + bli_sgemmsup_rd_zen5_asm_6x64m_col_stored_c, + sgemmGenericSUP, + ::testing::Combine( + ::testing::Values(gtint_t(64), // values of n, L64_N + gtint_t(48), // L48_N + gtint_t(32), // L32_N + gtint_t(8), // L8_N + gtint_t(7), // L4_N + L3_N + gtint_t(3), // L3_N + gtint_t(2), // L2_N + gtint_t(1), // L1_N + gtint_t(63)), // Combination of fringe cases for N + ::testing::Range(gtint_t(1), gtint_t(7), gtint_t(1)), // values of n(L6_M to L1_M) + ::testing::Values(gtint_t(64), // values of k, L64_K + gtint_t(32), // L32_K + gtint_t(16), // L16_K + gtint_t(12), // L_gt_8_K + gtint_t(7), // L_lt_8_K + gtint_t(256), // 4 * L64_K + gtint_t(303)), // Combination of main and fringe cases for K + ::testing::Values(2.0, 1.0, -1.0), // alpha value + ::testing::Values(1.0, 0.0, -1.0, 2.3), // beta value + ::testing::Values('c'), // storage of c + ::testing::Values(K_bli_sgemmsup_rd_zen5_asm_6x64m), // sgemm_sup_kernel + ::testing::Values(gtint_t(6)), // Micro kernel block MR + ::testing::Values('t'), // transa, has to be T for row storage + ::testing::Values('n'), // transb, has to be N for row storage + ::testing::Values(true), // kernel pref + ::testing::Values(true, false) // memory test + ), + ::sgemmGenericSUPPrint() + ); +#endif + + +#ifdef K_bli_sgemmsup_rd_zen5_asm_6x64n +INSTANTIATE_TEST_SUITE_P( + bli_sgemmsup_rd_zen5_asm_6x64n_row_stored_c, + sgemmGenericSUP, + ::testing::Combine( + ::testing::Range(gtint_t(1), gtint_t(7), 1), // values of m + ::testing::Range(gtint_t(1), gtint_t(65), 1), // values of n + ::testing::Values(gtint_t(64), // values of k, L64_K + gtint_t(32), // L32_K + gtint_t(16), // L16_K + gtint_t(12), // L_gt_8_K + gtint_t(7), // L_lt_8_K + gtint_t(256), // 4 * L64_K + gtint_t(303)), // Combination of main and fringe cases for K + ::testing::Values(2.0, 1.0, -1.0), // alpha value + ::testing::Values(1.0, 0.0, -1.0, 2.3), // beta value + ::testing::Values('r'), // storage of c + ::testing::Values(K_bli_sgemmsup_rd_zen5_asm_6x64n), // sgemm_sup_kernel + ::testing::Values(gtint_t(6)), // Micro kernel block MR + ::testing::Values('n'), // transa + ::testing::Values('t'), // transb + ::testing::Values(false), // kernel pref + ::testing::Values(true, false) // memory test + ), + ::sgemmGenericSUPPrint() + ); +#endif + +#endif // defined(BLIS_KERNELS_ZEN5) && defined(GTEST_AVX512) + /*******************************************************/ /* Native Kernel testing */ /*******************************************************/ diff --git a/kernels/zen5/3/bli_gemm_tiny_zen5.c b/kernels/zen5/3/bli_gemm_tiny_zen5.c new file mode 100644 index 000000000..338c77052 --- /dev/null +++ b/kernels/zen5/3/bli_gemm_tiny_zen5.c @@ -0,0 +1,101 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +// Defining separate static arrays to hold all the kernel info, based on the datatype +static gemmtiny_ukr_info_t cgemmtiny_ukr_zen5[] = +{ + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, FALSE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, TRUE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, FALSE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, FALSE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, FALSE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, TRUE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, FALSE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, FALSE, 24, 4 } +}; + +static gemmtiny_ukr_info_t zgemmtiny_ukr_zen5[] = +{ + { (void *)bli_zgemmsup_cv_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cd_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cv_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cv_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cv_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cd_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cv_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cv_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 } +}; + +static gemmtiny_ukr_info_t sgemmtiny_ukr_zen5[] = +{ + { (void *)bli_sgemmsup_rv_zen4_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rd_zen5_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rv_zen4_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rv_zen4_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rv_zen4_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rd_zen5_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rv_zen4_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rv_zen4_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 } +}; + +// Function macro that defines the bli_?gemmtiny_ukr_zen5_info functions +// These are used to acquire the kernel info at framework level +#undef GENTFUNC +#define GENTFUNC( ftype, ch, tfuncname ) \ +err_t PASTEMAC( ch, tfuncname ) \ + ( \ + stor3_t stor_id, \ + gemmtiny_ukr_info_t *fp_info \ + ) \ +{ \ + /* Acquire the object based on stor_id */ \ + *fp_info = TINY_GEMM_ZEN5(ch)[stor_id]; \ + /* If the kernel doesn't exist, return the appropriate signal */ \ + if ( fp_info->ukr_fp == NULL ) \ + { \ + return BLIS_NOT_YET_IMPLEMENTED; \ + } \ + else if ( ( fp_info->pack_fp == NULL ) && ( fp_info->enable_pack == TRUE ) ) \ + { \ + return BLIS_NOT_YET_IMPLEMENTED; \ + } \ +\ + return BLIS_SUCCESS; \ +} \ + +GENTFUNC( scomplex, c, gemmtiny_ukr_zen5_info ) +GENTFUNC( dcomplex, z, gemmtiny_ukr_zen5_info ) +GENTFUNC( float, s, gemmtiny_ukr_zen5_info ) \ No newline at end of file diff --git a/kernels/zen5/3/bli_gemm_tiny_zen5.h b/kernels/zen5/3/bli_gemm_tiny_zen5.h new file mode 100644 index 000000000..d12055c38 --- /dev/null +++ b/kernels/zen5/3/bli_gemm_tiny_zen5.h @@ -0,0 +1,59 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +// Macro to access the appropriate static array(that contains the kernel list), +// based on the datatype +#define TINY_GEMM_ZEN5(ch) ch ## gemmtiny_ukr_zen5 + +// Function macro signatures for bli_?gemmtiny_ukr_zen5_info functions +// These are used to acquire the kernel info at framework level +#undef GENTFUNC +#define GENTFUNC( ftype, ch, tfuncname ) \ +err_t PASTEMAC( ch, tfuncname ) \ + ( \ + stor3_t stor_id, \ + gemmtiny_ukr_info_t *fp_info \ + ); \ + +GENTFUNC( scomplex, c, gemmtiny_ukr_zen5_info ) +GENTFUNC( dcomplex, z, gemmtiny_ukr_zen5_info ) +GENTFUNC( float, s, gemmtiny_ukr_zen5_info ) + +/* Enabling the query for AVX512 kernels, based on the library's configuration */ +/* Minimum requirement is 'ZEN5' */ +#define LOOKUP_ZEN5_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) \ +{ \ + /* Call the appropriate function to query the AVX512 object info */ \ + ukr_support = PASTEMAC(ch, gemmtiny_ukr_zen5_info)( stor_id, &gemmtiny_ukr_info ); \ +} diff --git a/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.c b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.c new file mode 100644 index 000000000..ca717795c --- /dev/null +++ b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.c @@ -0,0 +1,5202 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +#define BLIS_ASM_SYNTAX_ATT +#include "bli_x86_asm_macros.h" + +#include "bli_gemmsup_rd_zen5_asm_s6x64.h" + +void bli_sgemmsup_rd_zen5_asm_5x64 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This file's x64 kernels only handle columns of C in groups of 4. + // Any remaining 1-3 columns of C are handled elsewhere. + // This kernel handles 5 rows of C over those 4-column groups. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns of C at a time; any remaining 1-3 columns of C + // are intentionally not handled here. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + uint64_t n_main_loop = n0 - n_left; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // This kernel only computes columns of C in the range [0, 4 * n_iter ). + // If n0 < 4, this kernel does no work and those columns of C are handled elsewhere. + if ( n_iter == 0 ) return; + + // Main 5x4 microkernel over the full left rectangle handled by this kernel: + // Handles 5 rows and columns in multiples of 4. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm28 ) // xmm28 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( n_main_loop ), r15 ) // r15 = n_main_loop + sub ( imm ( 4 ), r15 ) // jj = n_main_loop - 4 + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + lea ( mem ( r12, r15, 4 ), r12 ) // step to column jj within C + + lea ( mem ( , r15, 1 ), rsi ) // rsi = r15 = 4*jj; + imul ( r9, rsi ) // rsi *= cs_b; + lea ( mem ( rdx, rsi, 1 ), rdx ) // step to column jj within B + + mov ( r12, rcx ) // rcx = base of the current 5x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, r11, 4 ) ) // C row 4 + lea ( mem ( rcx, r11, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, r11, 1 ) ) // C row 3 + mov ( r14, rax ) // restart A at the top of this 4-col block + mov ( rdx, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm30 accumulate a 5x4 tile. + INIT_ACCUM_5x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 5 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 5x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 5 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 5x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + // Load one 16-float vector from each of the 5 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 5 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 5x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + label ( .POST_ACCUM ) + + // alpha is preloaded into xmm28 above the J-loop + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 5x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 5x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + // C_STOR then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 28, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR ( r11, 31, 4, 5, 6 ) // update the first 3 rows of the 5x4 tile + + // xmm22/xmm25 <- rows 3..4, cols 0..3. + ZMM_REDUCE_4 ( 20, 23, 26, 29, 22 ) // xmm22 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + ZMM_REDUCE_4 ( 21, 24, 27, 30, 25 ) // xmm25 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), sum ( zmm30 )] + + ALPHA_SCALE2 ( 28, 22, 25 ) // scale the next 2 rows by alpha + + C_STOR2_CONT ( r11, 31, 22, 25 ) // update the next 2 rows + + jmp ( .SDONE ) + + // Reduce the 5x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 28, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 22 ) // xmm22 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + ZMM_REDUCE_4 ( 21, 24, 27, 30, 25 ) // xmm25 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), sum ( zmm30 )] + + ALPHA_SCALE2 ( 28, 22, 25 ) // scale the next 2 rows by alpha + + C_STOR_BZ2_CONT ( r11, 22, 25 ) // store the next 2 rows + + label ( .SDONE ) + + sub ( imm ( 4 ), r15 ) + test ( r15, r15 ) + jns ( .SLOOP3X4J ) // iterate while jj >= 0 + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_main_loop] "m" ( n_main_loop ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm23", "ymm24", "ymm26", "ymm27", + "ymm29", "ymm30", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_4x64 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This file's x64 kernels only handle columns of C in groups of 4. + // Any remaining 1-3 columns of C are handled elsewhere. + // This kernel handles 4 rows of C over those 4-column groups. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns of C at a time; any remaining 1-3 columns of C + // are intentionally not handled here. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + uint64_t n_main_loop = n0 - n_left; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // This kernel only computes columns of C in the range [0, 4 * n_iter ). + // If n0 < 4, this kernel does no work and those columns of C are handled elsewhere. + if ( n_iter == 0 ) return; + + // Main 4x4 microkernel over the full rectangle handled by this kernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( n_main_loop ), r15 ) // r15 = n_main_loop + sub ( imm ( 4 ), r15 ) // jj = n_main_loop - 4 + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + lea ( mem ( r12, r15, 4 ), r12 ) // step to column jj within C + + lea ( mem ( , r15, 1 ), rsi ) // rsi = r15 = 4*jj; + imul ( r9, rsi ) // rsi *= cs_b; + lea ( mem ( rdx, rsi, 1 ), rdx ) // step to column jj within B + + mov ( r12, rcx ) // rcx = base of the current 4x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + lea ( mem ( rcx, r11, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, r11, 1 ) ) // C row 3 + mov ( r14, rax ) // restart A at the top of this 4-col block + mov ( rdx, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm29 accumulate a 4x4 tile. + INIT_ACCUM_4x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 4 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 4x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA4 ( 6, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA4 ( 7, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 4 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 4x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + // Load one 16-float vector from each of the 4 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 4 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 4x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 4x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR ( r11, 31, 4, 5, 6 ) // update the first 3 rows of the 4x4 tile + + // xmm21 <- row 3, cols 0..3. + ZMM_REDUCE_4 ( 20, 23, 26, 29, 21 ) // xmm21 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + + ALPHA_SCALE1 ( 30, 21 ) // scale the next row by alpha + + C_STOR1_CONT ( r11, 31, 21 ) // update the next row + + jmp ( .SDONE ) + + // Reduce the 4x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 21 ) // xmm21 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + + ALPHA_SCALE1 ( 30, 21 ) // scale the next row by alpha + + C_STOR_BZ1_CONT ( r11, 21 ) // store the next row + + label ( .SDONE ) + + sub ( imm ( 4 ), r15 ) + test ( r15, r15 ) + jns ( .SLOOP3X4J ) // iterate while jj >= 0 + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_main_loop] "m" ( n_main_loop ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm23", "ymm26", "ymm29", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_3x64 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This file's x64 kernels only handle columns of C in groups of 4. + // Any remaining 1-3 columns of C are handled elsewhere. + // This kernel handles 3 rows of C over those 4-column groups. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns of C at a time; any remaining 1-3 columns of C + // are intentionally not handled here. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + uint64_t n_main_loop = n0 - n_left; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // This kernel only computes columns of C in the range [0, 4 * n_iter ). + // If n0 < 4, this kernel does no work and those columns of C are handled elsewhere. + if ( n_iter == 0 ) return; + + // Main 3x4 microkernel over the full rectangle handled by this kernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( n_main_loop ), r15 ) // r15 = n_main_loop + sub ( imm ( 4 ), r15 ) // jj = n_main_loop - 4 + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + lea ( mem ( r12, r15, 4 ), r12 ) // step to column jj within C + + lea ( mem ( , r15, 1 ), rsi ) // rsi = r15 = 4*jj; + imul ( r9, rsi ) // rsi *= cs_b; + lea ( mem ( rdx, rsi, 1 ), rdx ) // step to column jj within B + + mov ( r12, rcx ) // rcx = base of the current 3x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + mov ( r14, rax ) // restart A at the top of this 4-col block + mov ( rdx, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm19 accumulate a 3x4 tile. + INIT_ACCUM_3x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA3 ( 25, 17, 18, 19 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA3 ( 25, 17, 18, 19 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 3x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR ( r11, 31, 4, 5, 6 ) // update the 3 rows of the 3x4 tile + + jmp ( .SDONE ) + + // Reduce the 3x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the 3 rows without reading C + + label ( .SDONE ) + + sub ( imm ( 4 ), r15 ) + test ( r15, r15 ) + jns ( .SLOOP3X4J ) // iterate while jj >= 0 + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_main_loop] "m" ( n_main_loop ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_2x64 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This file's x64 kernels only handle columns of C in groups of 4. + // Any remaining 1-3 columns of C are handled elsewhere. + // This kernel handles 2 rows of C over those 4-column groups. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns of C at a time; any remaining 1-3 columns of C + // are intentionally not handled here. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + uint64_t n_main_loop = n0 - n_left; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // This kernel only computes columns of C in the range [0, 4 * n_iter ). + // If n0 < 4, this kernel does no work and those columns of C are handled elsewhere. + if ( n_iter == 0 ) return; + + // Main 2x4 microkernel over the full rectangle handled by this kernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( n_main_loop ), r15 ) // r15 = n_main_loop + sub ( imm ( 4 ), r15 ) // jj = n_main_loop - 4 + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + lea ( mem ( r12, r15, 4 ), r12 ) // step to column jj within C + + lea ( mem ( , r15, 1 ), rsi ) // rsi = r15 = 4*jj; + imul ( r9, rsi ) // rsi *= cs_b; + lea ( mem ( rdx, rsi, 1 ), rdx ) // step to column jj within B + + mov ( r12, rcx ) // rcx = base of the current 2x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + mov ( r14, rax ) // restart A at the top of this 4-col block + mov ( rdx, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm18 accumulate a 2x4 tile. + INIT_ACCUM_2x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 2x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 2x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 2x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA2 ( 25, 17, 18 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA2 ( 25, 17, 18 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 2x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // xmm4/xmm5 <- rows 0..1, cols 0..3. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR2 ( r11, 31, 4, 5 ) // update the 2 rows of the 2x4 tile + + jmp ( .SDONE ) + + // Reduce the 2x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR_BZ2 ( r11, 4, 5 ) // store the 2 rows without reading C + + label ( .SDONE ) + + sub ( imm ( 4 ), r15 ) + test ( r15, r15 ) + jns ( .SLOOP3X4J ) // iterate while jj >= 0 + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_main_loop] "m" ( n_main_loop ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm17", "ymm18", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_1x64 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This file's x64 kernels only handle columns of C in groups of 4. + // Any remaining 1-3 columns of C are handled elsewhere. + // This kernel handles 1 row of C over those 4-column groups. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns of C at a time; any remaining 1-3 columns of C + // are intentionally not handled here. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + uint64_t n_main_loop = n0 - n_left; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // This kernel only computes columns of C in the range [0, 4 * n_iter ). + // If n0 < 4, this kernel does no work and those columns of C are handled elsewhere. + if ( n_iter == 0 ) return; + + // Main 1x4 microkernel over the full rectangle handled by this kernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( n_main_loop ), r15 ) // r15 = n_main_loop + sub ( imm ( 4 ), r15 ) // jj = n_main_loop - 4 + mov ( var ( abuf ), r14 ) // load base address of a + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + lea ( mem ( r12, r15, 4 ), r12 ) // step to column jj within C + + lea ( mem ( , r15, 1 ), rsi ) // rsi = r15 = 4*jj; + imul ( r9, rsi ) // rsi *= cs_b; + lea ( mem ( rdx, rsi, 1 ), rdx ) // step to column jj within B + + mov ( r12, rcx ) // rcx = base of the current 1x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + mov ( r14, rax ) // restart A at the top of this 4-col block + mov ( rdx, rbx ) // restart B at the top of this 4-col block + + // zmm8/zmm11/zmm14/zmm17 accumulate a 1x4 tile. + INIT_ACCUM_1x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from the A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from the row of A. + vmovups ( ( rax ), zmm0 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 1x4 accumulators. + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm0, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm0, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm13 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vfmadd231ps ( ( rbx ), zmm13, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm13, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm13, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm13, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm15 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vfmadd231ps ( ( rbx ), zmm15, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm15, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm15, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm15, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm18 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vfmadd231ps ( ( rbx ), zmm18, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm18, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm18, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm18, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from the row of A. + vmovups ( ( rax ), zmm0 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 1x4 accumulators. + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm0, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm0, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm13 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vfmadd231ps ( ( rbx ), zmm13, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm13, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm13, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm13, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from the row of A. + vmovups ( ( rax ), zmm15 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 1x4 accumulators. + vfmadd231ps ( ( rbx ), zmm15, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm15, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm15, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm15, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA1 ( 6, 8 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA1 ( 7, 11 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA1 ( 22, 14 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA1 ( 25, 17 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA1 ( 6, 8 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA1 ( 7, 11 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA1 ( 22, 14 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA1 ( 25, 17 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 1x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // xmm4 <- row 0, cols 0..3. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + + ALPHA_SCALE1 ( 30, 4 ) // scale the row by alpha + + C_STOR1 ( r11, 31, 4 ) // update the row of the 1x4 tile + + jmp ( .SDONE ) + + // Reduce the 1x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + + ALPHA_SCALE1 ( 30, 4 ) // scale the row by alpha + + C_STOR_BZ1 ( r11, 4 ) // store the row without reading C + + label ( .SDONE ) + + sub ( imm ( 4 ), r15 ) + test ( r15, r15 ) + jns ( .SLOOP3X4J ) // iterate while jj >= 0 + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_main_loop] "m" ( n_main_loop ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm6", + "ymm7", "ymm8", "ymm10", "ymm11", "ymm13", + "ymm14", "ymm17", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_5x3 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 5 rows and 3 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 5x3 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + mov ( imm ( 7 ), esi ) + kmovw ( esi, K ( 2 ) ) // k2 = mask for the 3 active C columns + + mov ( var ( abuf ), rax ) // load base address of a + mov ( var ( bbuf ), rbx ) // load base address of b + mov ( var ( cbuf ), rcx ) // load base address of c + + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, r11, 4 ) ) // C row 4 + lea ( mem ( rcx, r11, 2 ), rsi ) // r11 = rcx + 2 * rs_c + prefetchw0 ( mem ( rsi, r11, 1 ) ) // C row 3 + + INIT_ACCUM_5x3 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 5x3 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + ZMM_REDUCE_3 ( 10, 13, 16, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_MASKED ( r11, 31, 4, 5, 6 ) // update the first 3 rows of the 5x3 tile + + ZMM_REDUCE_3 ( 20, 23, 26, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 )] + ZMM_REDUCE_3 ( 21, 24, 27, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_MASKED2_CONT ( r11, 31, 17, 18 ) // update the next 2 rows + + jmp ( .SDONE ) + + // Reduce the 5x3 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + ZMM_REDUCE_3 ( 10, 13, 16, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_BZ_MASKED ( r11, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_3 ( 20, 23, 26, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 )] + ZMM_REDUCE_3 ( 21, 24, 27, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_BZ_MASKED2_CONT ( r11, 17, 18 ) // store the next 2 rows + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1", "k2" + ) +} + +void bli_sgemmsup_rd_zen5_asm_4x3 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 4 rows and 3 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 4x3 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + mov ( imm ( 7 ), esi ) + kmovw ( esi, K ( 2 ) ) // k2 = mask for the 3 active C columns + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + lea ( mem ( rcx, r11, 2 ), rsi ) // r11 = rcx + 2 * rs_c + prefetchw0 ( mem ( rsi, r11, 1 ) ) // C row 3 + + INIT_ACCUM_4x3 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 4x3 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the first 2 rows by alpha + + C_STOR_MASKED2 ( r11, 31, 4, 5 ) // update the first 2 rows of the 4x3 tile + + ZMM_REDUCE_3 ( 10, 13, 16, 17 ) // xmm17 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 )] + ZMM_REDUCE_3 ( 20, 23, 26, 18 ) // xmm18 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_MASKED2_CONT ( r11, 31, 17, 18 ) // update the next 2 rows + + jmp ( .SDONE ) + + // Reduce the 4x3 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the first 2 rows by alpha + + C_STOR_BZ_MASKED2 ( r11, 4, 5 ) // store the first 2 rows without reading C + + ZMM_REDUCE_3 ( 10, 13, 16, 17 ) // xmm17 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 )] + ZMM_REDUCE_3 ( 20, 23, 26, 18 ) // xmm18 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_BZ_MASKED2_CONT ( r11, 17, 18 ) // store the next 2 rows + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1", "k2" + ) +} + +void bli_sgemmsup_rd_zen5_asm_3x3 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 3 rows and 3 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 3x3 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + mov ( imm ( 7 ), esi ) + kmovw ( esi, K ( 2 ) ) // k2 = mask for the 3 active C columns + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + + INIT_ACCUM_3x3 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA3 ( 22, 14, 15, 16 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA3 ( 22, 14, 15, 16 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 3x3 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + ZMM_REDUCE_3 ( 10, 13, 16, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_MASKED ( r11, 31, 4, 5, 6 ) // update the 3 rows of the 3x3 tile + + jmp ( .SDONE ) + + // Reduce the 3x3 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + ZMM_REDUCE_3 ( 10, 13, 16, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_BZ_MASKED ( r11, 4, 5, 6 ) // store the 3 rows without reading C + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1", "k2" + ) +} + +void bli_sgemmsup_rd_zen5_asm_2x3 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 2 rows and 3 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 2x3 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + mov ( imm ( 7 ), esi ) // k2 = keep the first 3 lanes for 3 columns of C + kmovw ( esi, K ( 2 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + + INIT_ACCUM_2x3 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA2 ( 22, 14, 15 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA2 ( 22, 14, 15 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 2x3 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR_MASKED2 ( r11, 31, 4, 5 ) // update the 2x3 tile in C + + jmp ( .SDONE ) + + // Reduce the 2x3 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR_BZ_MASKED2 ( r11, 4, 5 ) // store the 2x3 tile without reading C + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1", "k2" + ) +} + +void bli_sgemmsup_rd_zen5_asm_1x3 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 1 row and 3 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 1x3 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + mov ( imm ( 7 ), esi ) // k2 = keep the first 3 lanes for 3 columns of C + kmovw ( esi, K ( 2 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + prefetchw0 ( mem ( rcx ) ) // C row 0 + + INIT_ACCUM_1x3 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm0, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm13 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm13, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm13, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm13, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm15 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm15, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm15, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm15, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm16 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm16, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm16, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm16, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm0, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm13 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm13, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm13, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm13, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + + add ( imm ( 16*4 ), rax ) + + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm0, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA1 ( 6, 8 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA1 ( 7, 11 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA1 ( 22, 14 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA1 ( 6, 8 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA1 ( 7, 11 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA1 ( 22, 14 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 1x3 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + + ALPHA_SCALE1 ( 30, 4 ) // scale the row by alpha + + C_STOR_MASKED1 ( r11, 31, 4 ) // update the 1x3 tile in C + + jmp ( .SDONE ) + + // Reduce the 1x3 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + + ALPHA_SCALE1 ( 30, 4 ) // scale the row by alpha + + C_STOR_BZ_MASKED1 ( r11, 4 ) // store the 1x3 tile without reading C + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1", "k2" + ) +} + +void bli_sgemmsup_rd_zen5_asm_5x2 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 5 rows and 2 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 5x2 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, r11, 4 ) ) // C row 4 + lea ( mem ( rcx, r11, 2 ), rsi ) // r11 = rcx + 2 * rs_c + prefetchw0 ( mem ( rsi, r11, 1 ) ) // C row 3 + + INIT_ACCUM_5x2 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 5x2 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + ZMM_REDUCE_2 ( 10, 13, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_2_FLOATS ( r11, 31, 4, 5, 6 ) // update the first 3 rows of the 5x2 tile + + ZMM_REDUCE_2 ( 20, 23, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 )] + ZMM_REDUCE_2 ( 21, 24, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_2_FLOATS2_CONT ( r11, 31, 17, 18 ) // update the next 2 rows + + jmp ( .SDONE ) + + // Reduce the 5x2 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + ZMM_REDUCE_2 ( 10, 13, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_BZ_2_FLOATS ( r11, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_2 ( 20, 23, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 )] + ZMM_REDUCE_2 ( 21, 24, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_BZ_2_FLOATS2_CONT ( r11, 17, 18 ) // store the next 2 rows + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_4x2 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 4 rows and 2 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 4x2 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + lea ( mem ( rcx, r11, 2 ), rsi ) // r11 = rcx + 2 * rs_c + prefetchw0 ( mem ( rsi, r11, 1 ) ) // C row 3 + + INIT_ACCUM_4x2 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 3x2 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the first 2 rows by alpha + + C_STOR_2_FLOATS2 ( r11, 31, 4, 5 ) // update the first 2 rows of the 4x2 tile + + ZMM_REDUCE_2 ( 10, 13, 17 ) // xmm17 = [sum ( zmm10 ), sum ( zmm13 )] + ZMM_REDUCE_2 ( 20, 23, 18 ) // xmm18 = [sum ( zmm20 ), sum ( zmm23 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_2_FLOATS2_CONT ( r11, 31, 17, 18 ) // update the next 2 rows + + jmp ( .SDONE ) + + // Reduce the 4x2 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the first 2 rows by alpha + + C_STOR_BZ_2_FLOATS2 ( r11, 4, 5 ) // store the first 2 rows without reading C + + ZMM_REDUCE_2 ( 10, 13, 17 ) // xmm17 = [sum ( zmm10 ), sum ( zmm13 )] + ZMM_REDUCE_2 ( 20, 23, 18 ) // xmm18 = [sum ( zmm20 ), sum ( zmm23 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_BZ_2_FLOATS2_CONT ( r11, 17, 18 ) // store the next 2 rows + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_3x2 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 3 rows and 2 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 3x2 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + + INIT_ACCUM_3x2 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 2x2 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + ZMM_REDUCE_2 ( 10, 13, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_2_FLOATS ( r11, 31, 4, 5, 6 ) // update the 3x2 tile in C + + jmp ( .SDONE ) + + // Reduce the 3x2 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + ZMM_REDUCE_2 ( 10, 13, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_BZ_2_FLOATS ( r11, 4, 5, 6 ) // store the 3x2 tile without reading C + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_2x2 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 2 rows and 2 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 2x2 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + + INIT_ACCUM_2x2 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 2x2 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR_2_FLOATS2 ( r11, 31, 4, 5 ) // update the 2x2 tile in C + + jmp ( .SDONE ) + + // Reduce the 2x2 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR_BZ_2_FLOATS2 ( r11, 4, 5 ) // store the 2x2 tile without reading C + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_1x2 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 1 row and 2 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 1x2 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + + INIT_ACCUM_1x2 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm12 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm12, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm12, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm13 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm13, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm13, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm14 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm14, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm14, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm12 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm12, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm12, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + + add ( imm ( 16*4 ), rax ) + + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA1 ( 6, 8 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA1 ( 7, 11 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA1 ( 6, 8 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA1 ( 7, 11 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 1x2 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + + ALPHA_SCALE1 ( 30, 4 ) // scale the row by alpha + + C_STOR_2_FLOATS1 ( r11, 31, 4 ) // update the 1x2 tile in C + + jmp ( .SDONE ) + + // Reduce the 1x2 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + + ALPHA_SCALE1 ( 30, 4 ) // scale the row by alpha + + C_STOR_BZ_2_FLOATS1 ( r11, 4 ) // store the 1x2 tile without reading C + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} diff --git a/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.h b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.h new file mode 100644 index 000000000..9e08b00a6 --- /dev/null +++ b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.h @@ -0,0 +1,700 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#define BLIS_ASM_SYNTAX_ATT +#include "bli_x86_asm_macros.h" + +/* Zero only accumulators for 4 B-columns (6x4 tile): zmm8-zmm31 */ +#define INIT_ACCUM_4COL \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm17,zmm17,zmm17 ) \ + vxorps( zmm18,zmm18,zmm18 ) \ + vxorps( zmm19,zmm19,zmm19 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm21,zmm21,zmm21 ) \ + vxorps( zmm22,zmm22,zmm22 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm24,zmm24,zmm24 ) \ + vxorps( zmm25,zmm25,zmm25 ) \ + vxorps( zmm26,zmm26,zmm26 ) \ + vxorps( zmm27,zmm27,zmm27 ) \ + vxorps( zmm28,zmm28,zmm28 ) \ + vxorps( zmm29,zmm29,zmm29 ) \ + vxorps( zmm30,zmm30,zmm30 ) \ + vxorps( zmm31,zmm31,zmm31 ) + +/* Zero only accumulators for 3 B-columns (6x3 tile): zmm8-16, zmm20-28 */ +#define INIT_ACCUM_3COL \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm21,zmm21,zmm21 ) \ + vxorps( zmm22,zmm22,zmm22 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm24,zmm24,zmm24 ) \ + vxorps( zmm25,zmm25,zmm25 ) \ + vxorps( zmm26,zmm26,zmm26 ) \ + vxorps( zmm27,zmm27,zmm27 ) \ + vxorps( zmm28,zmm28,zmm28 ) + +/* Zero only accumulators for 2 B-columns (6x2 tile): zmm8-13, zmm20-25 */ +#define INIT_ACCUM_2COL \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm21,zmm21,zmm21 ) \ + vxorps( zmm22,zmm22,zmm22 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm24,zmm24,zmm24 ) \ + vxorps( zmm25,zmm25,zmm25 ) + +/* 5-row x 4-B-col accumulators: zmm 8-10,11-13,14-16,17-19,20-21,23-24,26-27,29-30 (20 regs) */ +#define INIT_ACCUM_5x4 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm17,zmm17,zmm17 ) \ + vxorps( zmm18,zmm18,zmm18 ) \ + vxorps( zmm19,zmm19,zmm19 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm21,zmm21,zmm21 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm24,zmm24,zmm24 ) \ + vxorps( zmm26,zmm26,zmm26 ) \ + vxorps( zmm27,zmm27,zmm27 ) \ + vxorps( zmm29,zmm29,zmm29 ) \ + vxorps( zmm30,zmm30,zmm30 ) + +/* 4-row x 4-B-col accumulators: zmm 8-10,11-13,14-16,17-19,20,23,26,29 (16 regs) */ +#define INIT_ACCUM_4x4 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm17,zmm17,zmm17 ) \ + vxorps( zmm18,zmm18,zmm18 ) \ + vxorps( zmm19,zmm19,zmm19 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm26,zmm26,zmm26 ) \ + vxorps( zmm29,zmm29,zmm29 ) + +/* 3-row x 4-B-col accumulators: zmm 8-10,11-13,14-16,17-19 (12 regs) */ +#define INIT_ACCUM_3x4 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm17,zmm17,zmm17 ) \ + vxorps( zmm18,zmm18,zmm18 ) \ + vxorps( zmm19,zmm19,zmm19 ) + +/* 2-row x 4-B-col accumulators: zmm 8-9,11-12,14-15,17-18 (8 regs) */ +#define INIT_ACCUM_2x4 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm17,zmm17,zmm17 ) \ + vxorps( zmm18,zmm18,zmm18 ) + +/* 1-row x 4-B-col accumulators: zmm 8,11,14,17 (4 regs) */ +#define INIT_ACCUM_1x4 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm17,zmm17,zmm17 ) + +/* 5-row x 3-B-col accumulators: zmm 8-10,11-13,14-16,20-21,23-24,26-27 (15 regs) */ +#define INIT_ACCUM_5x3 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm21,zmm21,zmm21 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm24,zmm24,zmm24 ) \ + vxorps( zmm26,zmm26,zmm26 ) \ + vxorps( zmm27,zmm27,zmm27 ) + +/* 4-row x 3-B-col accumulators: zmm 8-10,11-13,14-16,20,23,26 (12 regs) */ +#define INIT_ACCUM_4x3 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm26,zmm26,zmm26 ) + +/* 3-row x 3-B-col accumulators: zmm 8-10,11-13,14-16 (9 regs) */ +#define INIT_ACCUM_3x3 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) + +/* 2-row x 3-B-col accumulators: zmm 8-9,11-12,14-15 (6 regs) */ +#define INIT_ACCUM_2x3 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) + +/* 1-row x 3-B-col accumulators: zmm 8,11,14 (3 regs) */ +#define INIT_ACCUM_1x3 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm14,zmm14,zmm14 ) + +/* 5-row x 2-B-col accumulators: zmm 8-10,11-13,20-21,23-24 (10 regs) */ +#define INIT_ACCUM_5x2 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm21,zmm21,zmm21 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm24,zmm24,zmm24 ) + +/* 4-row x 2-B-col accumulators: zmm 8-10,11-13,20,23 (8 regs) */ +#define INIT_ACCUM_4x2 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm23,zmm23,zmm23 ) + +/* 3-row x 2-B-col accumulators: zmm 8-10,11-13 (6 regs) */ +#define INIT_ACCUM_3x2 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) + +/* 2-row x 2-B-col accumulators: zmm 8-9,11-12 (4 regs) */ +#define INIT_ACCUM_2x2 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) + +/* 1-row x 2-B-col accumulators: zmm 8,11 (2 regs) */ +#define INIT_ACCUM_1x2 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm11,zmm11,zmm11 ) + +#define VFMA6( Rload, R0, R1, R2, R3, R4, R5 ) \ + vfmadd231ps( zmm0, zmm(Rload), zmm(R0) ) /* zmm(R0) += zmm0 * zmm(Rload) */ \ + vfmadd231ps( zmm1, zmm(Rload), zmm(R1) ) /* zmm(R1) += zmm1 * zmm(Rload) */ \ + vfmadd231ps( zmm2, zmm(Rload), zmm(R2) ) /* zmm(R2) += zmm2 * zmm(Rload) */ \ + vfmadd231ps( zmm3, zmm(Rload), zmm(R3) ) /* zmm(R3) += zmm3 * zmm(Rload) */ \ + vfmadd231ps( zmm4, zmm(Rload), zmm(R4) ) /* zmm(R4) += zmm4 * zmm(Rload) */ \ + vfmadd231ps( zmm5, zmm(Rload), zmm(R5) ) /* zmm(R5) += zmm5 * zmm(Rload) */ + +#define VFMA5( Rload, R0, R1, R2, R3, R4 ) \ + vfmadd231ps( zmm0, zmm(Rload), zmm(R0) ) /* zmm(R0) += zmm0 * zmm(Rload) */ \ + vfmadd231ps( zmm1, zmm(Rload), zmm(R1) ) /* zmm(R1) += zmm1 * zmm(Rload) */ \ + vfmadd231ps( zmm2, zmm(Rload), zmm(R2) ) /* zmm(R2) += zmm2 * zmm(Rload) */ \ + vfmadd231ps( zmm3, zmm(Rload), zmm(R3) ) /* zmm(R3) += zmm3 * zmm(Rload) */ \ + vfmadd231ps( zmm4, zmm(Rload), zmm(R4) ) /* zmm(R4) += zmm4 * zmm(Rload) */ + +#define VFMA4( Rload, R0, R1, R2, R3 ) \ + vfmadd231ps( zmm0, zmm(Rload), zmm(R0) ) /* zmm(R0) += zmm0 * zmm(Rload) */ \ + vfmadd231ps( zmm1, zmm(Rload), zmm(R1) ) /* zmm(R1) += zmm1 * zmm(Rload) */ \ + vfmadd231ps( zmm2, zmm(Rload), zmm(R2) ) /* zmm(R2) += zmm2 * zmm(Rload) */ \ + vfmadd231ps( zmm3, zmm(Rload), zmm(R3) ) /* zmm(R3) += zmm3 * zmm(Rload) */ + +#define VFMA3( Rload, R0, R1, R2 ) \ + vfmadd231ps( zmm0, zmm(Rload), zmm(R0) ) /* zmm(R0) += zmm0 * zmm(Rload) */ \ + vfmadd231ps( zmm1, zmm(Rload), zmm(R1) ) /* zmm(R1) += zmm1 * zmm(Rload) */ \ + vfmadd231ps( zmm2, zmm(Rload), zmm(R2) ) /* zmm(R2) += zmm2 * zmm(Rload) */ + +#define VFMA2( Rload, R0, R1 ) \ + vfmadd231ps( zmm0, zmm(Rload), zmm(R0) ) /* zmm(R0) += zmm0 * zmm(Rload) */ \ + vfmadd231ps( zmm1, zmm(Rload), zmm(R1) ) /* zmm(R1) += zmm1 * zmm(Rload) */ + +#define VFMA1( Rload, R0 ) \ + vfmadd231ps( zmm0, zmm(Rload), zmm(R0) ) /* zmm(R0) += zmm0 * zmm(Rload) */ + +/* zmm(R0)=[a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15] */ +/* zmm(R1)=[b0,b1,b2,b3,b4,b5,b6,b7,b8,b9,b10,b11,b12,b13,b14,b15] */ +/* zmm(R2)=[c0,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15] */ +/* zmm(R3)=[d0,d1,d2,d3,d4,d5,d6,d7,d8,d9,d10,d11,d12,d13,d14,d15] */ +/* Input: four 16-float accumulators in zmm(R0) through zmm(R3). */ +/* Output: four final sums packed as [sum(a),sum(b),sum(c),sum(d)] in xmm(RD). */ +#define ZMM_REDUCE_4(R0, R1, R2, R3, RD) \ + vshufps( imm(0x88), zmm(R1), zmm(R0), zmm0 ) /* zmm0=[a0,a2,b0,b2,a4,a6,b4,b6, */ \ + /* a8,a10,b8,b10,a12,a14,b12,b14] */ \ + vshufps( imm(0xDD), zmm(R1), zmm(R0), zmm1 ) /* zmm1=[a1,a3,b1,b3,a5,a7,b5,b7, */ \ + /* a9,a11,b9,b11,a13,a15,b13,b15] */ \ + vaddps( zmm1, zmm0, zmm0 ) /* zmm0=[a0+a1,a2+a3,b0+b1,b2+b3, */ \ + /* a4+a5,a6+a7,b4+b5,b6+b7, */ \ + /* a8+a9,a10+a11,b8+b9,b10+b11, */ \ + /* a12+a13,a14+a15,b12+b13,b14+b15] */ \ + vshufps( imm(0x88), zmm(R3), zmm(R2), zmm2 ) /* zmm2=[c0,c2,d0,d2,c4,c6,d4,d6, */ \ + /* c8,c10,d8,d10,c12,c14,d12,d14] */ \ + vshufps( imm(0xDD), zmm(R3), zmm(R2), zmm3 ) /* zmm3=[c1,c3,d1,d3,c5,c7,d5,d7, */ \ + /* c9,c11,d9,d11,c13,c15,d13,d15] */ \ + vaddps( zmm3, zmm2, zmm2 ) /* zmm2=[c0+c1,c2+c3,d0+d1,d2+d3, */ \ + /* c4+c5,c6+c7,d4+d5,d6+d7, */ \ + /* c8+c9,c10+c11,d8+d9,d10+d11, */ \ + /* c12+c13,c14+c15,d12+d13,d14+d15] */ \ + vshufps( imm(0x88), zmm2, zmm0, zmm1 ) /* zmm1=[a0+a1,b0+b1,c0+c1,d0+d1, */ \ + /* a4+a5,b4+b5,c4+c5,d4+d5, */ \ + /* a8+a9,b8+b9,c8+c9,d8+d9, */ \ + /* a12+a13,b12+b13,c12+c13,d12+d13] */ \ + vshufps( imm(0xDD), zmm2, zmm0, zmm3 ) /* zmm3=[a2+a3,b2+b3,c2+c3,d2+d3, */ \ + /* a6+a7,b6+b7,c6+c7,d6+d7, */ \ + /* a10+a11,b10+b11,c10+c11,d10+d11, */ \ + /* a14+a15,b14+b15,c14+c15,d14+d15] */ \ + vaddps( zmm3, zmm1, zmm1 ) /* zmm1=[a0+...+a3,b0+...+b3,c0+...+c3,d0+...+d3, */ \ + /* a4+...+a7,b4+...+b7,c4+...+c7,d4+...+d7, */ \ + /* a8+...+a11,b8+...+b11,c8+...+c11,d8+...+d11, */ \ + /* a12+...+a15,b12+...+b15,c12+...+c15,d12+...+d15] */ \ + /* zmm1 now holds four quarter-sums for each output row. */ \ + vextractf32x8( imm(0x01), zmm1, ymm0 ) /* ymm0=[a8+...+a11,b8+...+b11,c8+...+c11,d8+...+d11, */ \ + /* a12+...+a15,b12+...+b15,c12+...+c15,d12+...+d15] */ \ + vaddps( ymm1, ymm0, ymm0 ) /* ymm0=[a0+...+a3+a8+...+a11,b0+...+b3+b8+...+b11, */ \ + /* c0+...+c3+c8+...+c11,d0+...+d3+d8+...+d11, */ \ + /* a4+...+a7+a12+...+a15,b4+...+b7+b12+...+b15, */ \ + /* c4+...+c7+c12+...+c15,d4+...+d7+d12+...+d15] */ \ + vextractf128( imm(0x01), ymm0, xmm1 ) /* xmm1=[a4+...+a7+a12+...+a15,b4+...+b7+b12+...+b15, */ \ + /* c4+...+c7+c12+...+c15,d4+...+d7+d12+...+d15] */ \ + vaddps( xmm0, xmm1, xmm(RD) ) /* xmm(RD)=[sum(a),sum(b),sum(c),sum(d)] */ + +/* Input: three 16-float accumulators in zmm(R0), zmm(R1), and zmm(R2). */ +/* Output: [sum(a),sum(b),sum(c),sum(c)] in xmm(RD). */ +#define ZMM_REDUCE_3(R0, R1, R2, RD) \ + vshufps( imm(0x88), zmm(R1), zmm(R0), zmm0 ) /* zmm0=[a0,a2,b0,b2,a4,a6,b4,b6, */ \ + /* a8,a10,b8,b10,a12,a14,b12,b14] */ \ + vshufps( imm(0xDD), zmm(R1), zmm(R0), zmm1 ) /* zmm1=[a1,a3,b1,b3,a5,a7,b5,b7, */ \ + /* a9,a11,b9,b11,a13,a15,b13,b15] */ \ + vaddps( zmm1, zmm0, zmm0 ) /* zmm0=[a0+a1,a2+a3,b0+b1,b2+b3, */ \ + /* a4+a5,a6+a7,b4+b5,b6+b7, */ \ + /* a8+a9,a10+a11,b8+b9,b10+b11, */ \ + /* a12+a13,a14+a15,b12+b13,b14+b15] */ \ + vshufps( imm(0x88), zmm(R2), zmm(R2), zmm2 ) /* zmm2=[c0,c2,c0,c2,c4,c6,c4,c6, */ \ + /* c8,c10,c8,c10,c12,c14,c12,c14] */ \ + vshufps( imm(0xDD), zmm(R2), zmm(R2), zmm3 ) /* zmm3=[c1,c3,c1,c3,c5,c7,c5,c7, */ \ + /* c9,c11,c9,c11,c13,c15,c13,c15] */ \ + vaddps( zmm3, zmm2, zmm2 ) /* zmm2=[c0+c1,c2+c3,c0+c1,c2+c3, */ \ + /* c4+c5,c6+c7,c4+c5,c6+c7, */ \ + /* c8+c9,c10+c11,c8+c9,c10+c11, */ \ + /* c12+c13,c14+c15,c12+c13,c14+c15] */ \ + vshufps( imm(0x88), zmm2, zmm0, zmm1 ) /* zmm1=[a0+a1,b0+b1,c0+c1,c0+c1, */ \ + /* a4+a5,b4+b5,c4+c5,c4+c5, */ \ + /* a8+a9,b8+b9,c8+c9,c8+c9, */ \ + /* a12+a13,b12+b13,c12+c13,c12+c13] */ \ + vshufps( imm(0xDD), zmm2, zmm0, zmm3 ) /* zmm3=[a2+a3,b2+b3,c2+c3,c2+c3, */ \ + /* a6+a7,b6+b7,c6+c7,c6+c7, */ \ + /* a10+a11,b10+b11,c10+c11,c10+c11, */ \ + /* a14+a15,b14+b15,c14+c15,c14+c15] */ \ + vaddps( zmm3, zmm1, zmm1 ) /* zmm1=[a0+...+a3,b0+...+b3,c0+...+c3,c0+...+c3, */ \ + /* a4+...+a7,b4+...+b7,c4+...+c7,c4+...+c7, */ \ + /* a8+...+a11,b8+...+b11,c8+...+c11,c8+...+c11, */ \ + /* a12+...+a15,b12+...+b15,c12+...+c15,c12+...+c15] */ \ + vextractf32x8( imm(0x01), zmm1, ymm0 ) /* ymm0=[a8+...+a11,b8+...+b11,c8+...+c11,c8+...+c11, */ \ + /* a12+...+a15,b12+...+b15,c12+...+c15,c12+...+c15] */ \ + vaddps( ymm1, ymm0, ymm0 ) /* ymm0=[a0+...+a3+a8+...+a11,b0+...+b3+b8+...+b11, */ \ + /* c0+...+c3+c8+...+c11,c0+...+c3+c8+...+c11, */ \ + /* a4+...+a7+a12+...+a15,b4+...+b7+b12+...+b15, */ \ + /* c4+...+c7+c12+...+c15,c4+...+c7+c12+...+c15] */ \ + vextractf128( imm(0x01), ymm0, xmm1 ) /* xmm1=[a4+...+a7+a12+...+a15,b4+...+b7+b12+...+b15, */ \ + /* c4+...+c7+c12+...+c15,c4+...+c7+c12+...+c15] */ \ + vaddps( xmm0, xmm1, xmm(RD) ) /* xmm(RD)=[sum(a),sum(b),sum(c),sum(c)] */ + +/* Input: two 16-float accumulators in zmm(R0) and zmm(R1). */ +/* Output: [sum(a),sum(b),sum(a),sum(b)] in xmm(RD). */ +#define ZMM_REDUCE_2(R0, R1, RD) \ + vshufps( imm(0x88), zmm(R1), zmm(R0), zmm0 ) /* zmm0=[a0,a2,b0,b2,a4,a6,b4,b6, */ \ + /* a8,a10,b8,b10,a12,a14,b12,b14] */ \ + vshufps( imm(0xDD), zmm(R1), zmm(R0), zmm1 ) /* zmm1=[a1,a3,b1,b3,a5,a7,b5,b7, */ \ + /* a9,a11,b9,b11,a13,a15,b13,b15] */ \ + vaddps( zmm1, zmm0, zmm0 ) /* zmm0=[a0+a1,a2+a3,b0+b1,b2+b3, */ \ + /* a4+a5,a6+a7,b4+b5,b6+b7, */ \ + /* a8+a9,a10+a11,b8+b9,b10+b11, */ \ + /* a12+a13,a14+a15,b12+b13,b14+b15] */ \ + vextractf32x8( imm(0x01), zmm0, ymm1 ) /* ymm1=[a8+a9,a10+a11,b8+b9,b10+b11, */ \ + /* a12+a13,a14+a15,b12+b13,b14+b15] */ \ + vaddps( ymm0, ymm1, ymm0 ) /* ymm0=[a0+a1+a8+a9,a2+a3+a10+a11, */ \ + /* b0+b1+b8+b9,b2+b3+b10+b11, */ \ + /* a4+a5+a12+a13,a6+a7+a14+a15, */ \ + /* b4+b5+b12+b13,b6+b7+b14+b15] */ \ + vextractf128( imm(0x01), ymm0, xmm1 ) /* xmm1=[a4+a5+a12+a13,a6+a7+a14+a15, */ \ + /* b4+b5+b12+b13,b6+b7+b14+b15] */ \ + vaddps( xmm0, xmm1, xmm0 ) /* xmm0=[a0+a1+a4+a5+a8+a9+a12+a13, */ \ + /* a2+a3+a6+a7+a10+a11+a14+a15, */ \ + /* b0+b1+b4+b5+b8+b9+b12+b13, */ \ + /* b2+b3+b6+b7+b10+b11+b14+b15] */ \ + vshufps( imm(0x88), xmm0, xmm0, xmm(RD) ) /* xmm(RD)=[a0+a1+a4+a5+a8+a9+a12+a13, */ \ + /* b0+b1+b4+b5+b8+b9+b12+b13, */ \ + /* a0+a1+a4+a5+a8+a9+a12+a13, */ \ + /* b0+b1+b4+b5+b8+b9+b12+b13] */ \ + vshufps( imm(0xDD), xmm0, xmm0, xmm3 ) /* xmm3=[a2+a3+a6+a7+a10+a11+a14+a15, */ \ + /* b2+b3+b6+b7+b10+b11+b14+b15, */ \ + /* a2+a3+a6+a7+a10+a11+a14+a15, */ \ + /* b2+b3+b6+b7+b10+b11+b14+b15] */ \ + vaddps( xmm3, xmm(RD), xmm(RD) ) /* xmm(RD)=[sum(a),sum(b),sum(a),sum(b)] */ + +#define C_STOR_BZ_2_FLOATS(R_rs_c, R0, R1, R2) \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovlps( xmm(R2), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R2)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_2_FLOATS1(R_rs_c, R0) \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_2_FLOATS2(R_rs_c, R0, R1) \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_MASKED(R_rs_c, R0, R1, R2) \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R2), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_MASKED2(R_rs_c, R0, R1) \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_MASKED1(R_rs_c, R0) \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ(R_rs_c, R0, R1, R2) \ + vmovups( xmm(R0), mem( rcx ) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem( rcx ) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R2), mem( rcx ) ) /* mem(rcx) <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ2(R_rs_c, R0, R1) \ + vmovups( xmm(R0), mem( rcx ) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem( rcx ) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ1(R_rs_c, R0) \ + vmovups( xmm(R0), mem( rcx ) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_CONT(R_rs_c, R0, R1, R2) \ + vmovups( xmm(R0), mem( rcx ) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem( rcx ) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R2), mem( rcx ) ) /* mem(rcx) <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ2_CONT(R_rs_c, R0, R1) \ + vmovups( xmm(R0), mem( rcx ) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem( rcx ) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ1_CONT(R_rs_c, R0) \ + vmovups( xmm(R0), mem( rcx ) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_MASKED_CONT(R_rs_c, R0, R1, R2) \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R2), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_MASKED2_CONT(R_rs_c, R0, R1) \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_2_FLOATS_CONT(R_rs_c, R0, R1, R2) \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovlps( xmm(R2), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R2)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_2_FLOATS2_CONT(R_rs_c, R0, R1) \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define ALPHA_SCALE(Ralpha, R0, R1, R2) \ + vmulps( xmm(Ralpha), xmm(R0), xmm(R0) ) /* xmm(R0) = alpha * xmm(R0) */ \ + vmulps( xmm(Ralpha), xmm(R1), xmm(R1) ) /* xmm(R1) = alpha * xmm(R1) */ \ + vmulps( xmm(Ralpha), xmm(R2), xmm(R2) ) /* xmm(R2) = alpha * xmm(R2) */ + +#define C_STOR(R_rs_c, Rbeta, R0, R1, R2) \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), (rcx) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), (rcx) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R2) ) /* xmm(R2) = beta * C + xmm(R2) */ \ + vmovups( xmm(R2), (rcx) ) /* mem(rcx) <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_CONT(R_rs_c, Rbeta, R0, R1, R2) \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), (rcx) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), (rcx) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R2) ) /* xmm(R2) = beta * C + xmm(R2) */ \ + vmovups( xmm(R2), (rcx) ) /* mem(rcx) <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_MASKED(R_rs_c, Rbeta, R0, R1, R2) \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R2) ) /* xmm(R2) = beta * C + xmm(R2) */ \ + vmovups( xmm(R2), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_MASKED_CONT(R_rs_c, Rbeta, R0, R1, R2) \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R2) ) /* xmm(R2) = beta * C + xmm(R2) */ \ + vmovups( xmm(R2), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_2_FLOATS(R_rs_c, Rbeta, R0, R1, R2) \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R2) ) /* xmm(R2) = beta * C + xmm(R2) */ \ + vmovlps( xmm(R2), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R2)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_2_FLOATS_CONT(R_rs_c, Rbeta, R0, R1, R2) \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R2) ) /* xmm(R2) = beta * C + xmm(R2) */ \ + vmovlps( xmm(R2), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R2)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define ALPHA_SCALE2(Ralpha, R0, R1) \ + vmulps( xmm(Ralpha), xmm(R0), xmm(R0) ) /* xmm(R0) = alpha * xmm(R0) */ \ + vmulps( xmm(Ralpha), xmm(R1), xmm(R1) ) /* xmm(R1) = alpha * xmm(R1) */ + +#define ALPHA_SCALE1(Ralpha, R0) \ + vmulps( xmm(Ralpha), xmm(R0), xmm(R0) ) /* xmm(R0) = alpha * xmm(R0) */ + +#define C_STOR2_CONT(R_rs_c, Rbeta, R0, R1) \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), (rcx) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), (rcx) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR1_CONT(R_rs_c, Rbeta, R0) \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), (rcx) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_MASKED2(R_rs_c, Rbeta, R0, R1) \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_MASKED2_CONT(R_rs_c, Rbeta, R0, R1) \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_2_FLOATS2(R_rs_c, Rbeta, R0, R1) \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_2_FLOATS2_CONT(R_rs_c, Rbeta, R0, R1) \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR2(R_rs_c, Rbeta, R0, R1) \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), (rcx) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), (rcx) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR1(R_rs_c, Rbeta, R0) \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), (rcx) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_MASKED1(R_rs_c, Rbeta, R1) \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_2_FLOATS1(R_rs_c, Rbeta, R0) \ + vmovsd( mem(rcx), xmm0 ) /* xmm0 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm0, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ diff --git a/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64m.c b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64m.c new file mode 100644 index 000000000..ac8b46a84 --- /dev/null +++ b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64m.c @@ -0,0 +1,1712 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +#define BLIS_ASM_SYNTAX_ATT +#include "bli_x86_asm_macros.h" + +#include "bli_gemmsup_rd_zen5_asm_s6x64.h" + +void bli_sgemmsup_rd_zen5_asm_6x64m + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // Main loop handles 4 columns at a time; 1-3 columns fall through to n_left dispatch. + uint64_t n_left = n0 % 4; + + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + uint64_t b_step_base = 0; + + uint64_t n_iter = n0 / 4; + uint64_t n_main_loop = n0 - n_left; + uint64_t j_step_end = 0; + uint64_t j_step_start = n_main_loop >= 1 ? ( ( n_main_loop - 1 ) / 16 ) * 16 : 0; + + // Main loop handles 6 rows at a time; 1-5 rows fall through to m_left dispatch. + uint64_t m_iter = m0 / 6; + uint64_t m_left = m0 % 6; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + // The decomposition into edge kernels is handled as follows: + // Example, m0 = 7 and n0 = 5: + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ m_left ][ n_left ] + // The asm handles all multiples of 6 rows and 4 columns in the top-left of the matrix + // Next, any remaining m_left rows are handled by a separate edge microkernel after the + // assembly section which still processes 4 columns at a time in the n dimension. + // Finally, any remaining n_left columns are handled by a separate edge microkernel + // that processes the remaining right side columns. + // n_left dispatch: handle the right edge columns that remain after the asm + // kernel covers all full 4-column blocks. + // This path owns the entire right-edge strip across rows [0, m0 ), including + // the bottom-right corner that the x64 m_left kernels intentionally skip. + // Computes the n_left part of C. + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + if ( n_left ) + { + float* restrict cij = c; + float* restrict bj = b; + float* restrict ai = a; + + // Start at column 4 * n_iter, i.e. the first column not covered by asm. + cij += n_main_loop*cs_c0; + bj += n_main_loop*cs_b0; + + if ( 3 == n_left ) + { + const dim_t nr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_6x3m + ( + conja, conjb, m0, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == n_left ) + { + const dim_t nr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_6x2m + ( + conja, conjb, m0, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == n_left ) + { + bli_sgemv_ex + ( + BLIS_NO_TRANSPOSE, conjb, m0, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, + beta, cij, rs_c0, cntx, NULL + ); + } + } + + // The asm microkernel only handles full 6x4 tiles. + // If either dimension is already in edge territory, skip straight to the + // C dispatch below. + if ( m_iter == 0 || n_iter == 0 ) goto consider_edge_cases; + + // Main 6x4 microkernel over the full top-left rectangle: + // Handles all rows in multiples of 6, and columns in multiples of 4. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ][ asm ][ - ] + // [ asm ][ asm ][ asm ][ asm ][ - ] + // [ asm ][ asm ][ asm ][ asm ][ - ] + // [ asm ][ asm ][ asm ][ asm ][ - ] + // [ asm ][ asm ][ asm ][ asm ][ - ] + // [ asm ][ asm ][ asm ][ asm ][ - ] + // [ - ][ - ] + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + // Outer J-step loop: process 16 columns ( 4 groups of 4 ) at a time. + // For each 16-col step, sweep all m_iter row-blocks ( full SLOOP3X4I ), + // then advance to next 16-col step. This is done so that writes happen to the full + // cache line in the C matrix + label ( .SLOOP3X4J_STEP ) // Outer loop: 16-col steps + mov ( var ( j_step_start ), r15 ) // r15 = j_step_start + mov ( imm ( 16 ), rsi ) // rsi = 16 + add ( r15, rsi ) // rsi = j_step_start + 16 ( desired end ) + mov ( var ( n_main_loop ), rax ) // rax = n_main_loop ( max bound ) + cmp ( rax, rsi ) // compare rsi vs n_main_loop + cmovg ( rax, rsi ) // if rsi > n_main_loop, rsi = n_main_loop + mov ( rsi, var ( j_step_end ) ) // j_step_end = min ( j_step_start + 16, n_main_loop ) + + mov ( var ( m_iter ), r11 ) // ii = m_iter; + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( cbuf ), r12 ) // load base address of c + + // Compute B base for this 16-col step once, outside the I-loop. + mov ( var ( bbuf ), rdx ) + sub ( imm ( 4 ), rsi ) // rsi = j_step_end - 4 + imul ( r9, rsi ) // rsi = ( j_step_end - 4 ) * cs_b_bytes + add ( rsi, rdx ) // rdx = B base for rightmost 4-col group + mov ( rdx, var ( b_step_base ) ) // save B base for I-loop reuse + + label ( .SLOOP3X4I ) // Middle loop over 6-row blocks of C/A + + // B base for this 16-col step: bbuf + j_step_start * cs_b_bytes + mov ( var ( b_step_base ), rdx ) // reload B base for the beginning of the I-loop + mov ( var ( j_step_end ), r15 ) + sub ( imm ( 4 ), r15 ) // r15 = j_step_end - 4 + + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + // rcx = base of the current 6x4 output tile in C. + lea ( mem ( r12, r15, 4 ), rcx ) // rcx = c_row_base + jj * sizeof ( float ) + + // Prefetch the rows of C that will be written to in this loop + mov ( var ( rs_c ), rsi ) // rsi = rs_c + lea ( mem ( , rsi, 4 ), rsi ) // rsi = rs_c * sizeof ( float ) + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, rsi, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, rsi, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, rsi, 4 ) ) // C row 4 + lea ( mem ( rcx, rsi, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, rsi, 1 ) ) // C row 3 = rcx + 3 * rs_c + lea ( mem ( rcx, rsi, 4 ), rax ) // rax = rcx + 4 * rs_c + prefetchw0 ( mem ( rax, rsi, 1 ) ) // C row 5 = rcx + 5 * rs_c + + mov ( r14, rax ) // restart A at the top of this row-block + mov ( rdx, rbx ) // load b to rbx + lea ( mem ( r10, r8, 2 ), rdi ) // rdi = 5 * rs_a + + // zmm8-zmm31 accumulate a 6x4 tile. + INIT_ACCUM_4COL + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 6x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 6x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), ZMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), YMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + label ( .POST_ACCUM ) + + // Preload alpha into xmm7 to avoid redundant memory reloads. + // xmm7 survives ZMM_REDUCE calls ( they only clobber zmm0-3 ). + mov ( var ( alpha ), rax ) // load address of alpha + vbroadcastss ( ( rax ), xmm7 ) // xmm7 = alpha + mov ( var ( beta ), rax ) // load address of beta + vbroadcastss ( ( rax ), xmm0 ) + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm0 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 6x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 6x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + // C_STOR then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 7, 4, 5, 6 ) // scale the first 3 rows by alpha + + // Preload beta into xmm9 now that zmm9 is free ( consumed by ZMM_REDUCE_4 ). + mov ( var ( beta ), rax ) + vbroadcastss ( ( rax ), xmm9 ) // xmm9 = beta + mov ( var ( rs_c ), rdi ) // rdi = rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rdi = rs_c * sizeof ( float ) + + C_STOR ( rdi, 9, 4, 5, 6 ) // update the first 3 rows of the 6x4 tile + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + ZMM_REDUCE_4 ( 21, 24, 27, 30, 11 ) // xmm11 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), sum ( zmm30 )] + ZMM_REDUCE_4 ( 22, 25, 28, 31, 14 ) // xmm14 = [sum ( zmm22 ), sum ( zmm25 ), sum ( zmm28 ), sum ( zmm31 )] + + ALPHA_SCALE ( 7, 8, 11, 14 ) // scale the next 3 rows by alpha + + C_STOR_CONT ( rdi, 9, 8, 11, 14 ) // update the next 3 rows + + jmp ( .SDONE ) + + // Reduce the 6x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 7, 4, 5, 6 ) // scale the first 3 rows by alpha + + mov ( var ( rs_c ), rdi ) // rdi = rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rdi = rs_c * sizeof ( float ) + C_STOR_BZ ( rdi, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + ZMM_REDUCE_4 ( 21, 24, 27, 30, 11 ) // xmm11 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), sum ( zmm30 )] + ZMM_REDUCE_4 ( 22, 25, 28, 31, 14 ) // xmm14 = [sum ( zmm22 ), sum ( zmm25 ), sum ( zmm28 ), sum ( zmm31 )] + + ALPHA_SCALE ( 7, 8, 11, 14 ) // scale the next 3 rows by alpha + + C_STOR_BZ_CONT ( rdi, 8, 11, 14 ) // store the next 3 rows + + label ( .SDONE ) + + // Advance to the previous 4-column block of B/C ( R-to-L within block ). + lea ( mem ( , r9, 4 ), rsi ) // rsi = 4 * cs_b_bytes + sub ( rsi, rdx ) // rdx -= 4 * cs_b + sub ( imm ( 4 ), r15 ) + cmp ( var ( j_step_start ), r15 ) + jge ( .SLOOP3X4J ) // iterate within current 16-col step + + // Finished all 4 J-iterations for this row-block; step A and C down by 6 rows. + mov ( var ( rs_c ), rdi ) // load rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rs_c *= sizeof ( float ) => rs_c *= 4 + lea ( mem ( r12, rdi, 2 ), r12 ) + lea ( mem ( r12, rdi, 4 ), r12 ) // c_ii = r12 += 6*rs_c + + lea ( mem ( r14, r8, 2 ), r14 ) + lea ( mem ( r14, r8, 4 ), r14 ) // a_ii = r14 += 6*rs_a + + dec ( r11 ) + jne ( .SLOOP3X4I ) // iterate again if ii != 0. + + // Advance j_step_start by 16 and loop if more columns remain. + mov ( var ( j_step_start ), r15 ) + sub ( imm ( 16 ), r15 ) + mov ( r15, var ( j_step_start ) ) + test ( r15, r15 ) + jns ( .SLOOP3X4J_STEP ) // loop while j_step_start >= 0 i.e. if it is not-signed + + end_asm ( + : // output operands + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_main_loop] "m" ( n_main_loop ), + [m_iter] "m" ( m_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ), + [j_step_start] "m" ( j_step_start ), + [j_step_end] "m" ( j_step_end ), + [b_step_base] "m" ( b_step_base ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) + + consider_edge_cases: + + // Control reaches here in two cases: + // 1. asm finished the full-tile rectangle and only edge work remains. + // 2. asm was skipped because m0 < 6 or n0 < 4. + // m_left dispatch: handle the bottom edge rows that remain after the asm + // kernel covers all full 6-row blocks. + // The x64 callees receive n0, but internally they only compute the 4-col + // columns [0, 4 * n_iter ). They do not consume the 1-3 column tail. + // Computes the m_left part of C. + // [ - ][ - ][ - ][ - ][ - ] + // [ - ][ - ][ - ][ - ][ - ] + // [ - ][ - ][ - ][ - ][ - ] + // [ - ][ - ][ - ][ - ][ - ] + // [ - ][ - ][ - ][ - ][ - ] + // [ - ][ - ][ - ][ - ][ - ] + // [ m_left ][ - ] + if ( m_left && n_iter ) + { + const dim_t nr_cur = n0; + const dim_t i_edge = m0 - ( dim_t )m_left; + + // Start at row 6 * m_iter, i.e. the first row not covered by asm. + float* restrict cij = c + i_edge*rs_c; + float* restrict bj = b; + float* restrict ai = a + i_edge*rs_a; + + if ( 5 == m_left ) + { + dim_t mr_cur = 5; + bli_sgemmsup_rd_zen5_asm_5x64 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 4 == m_left ) + { + const dim_t mr_cur = 4; + + bli_sgemmsup_rd_zen5_asm_4x64 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 3 == m_left ) + { + const dim_t mr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_3x64 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == m_left ) + { + const dim_t mr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_2x64 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == m_left ) + { + const dim_t mr_cur = 1; + + bli_sgemmsup_rd_zen5_asm_1x64 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + } +} + +void bli_sgemmsup_rd_zen5_asm_6x3m + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 6 rows and 3 columns of C. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 6 rows at a time; 1-5 rows fall through to m_left dispatch. + uint64_t m_iter = m0 / 6; + uint64_t m_left = m0 % 6; + + uint64_t rs_a = rs_a0; + + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // The asm microkernel only handles full 6x3 tiles. + // If m0 < 6, skip straight to the C dispatch below. + if ( m_iter == 0 ) goto consider_edge_cases; + + // Main 6x3 microkernel over the full 6x3 rectangle: + // Handles all rows in multiples of 6. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ] + // [ asm ][ asm ][ asm ] + // [ asm ][ asm ][ asm ] + // [ asm ][ asm ][ asm ] + // [ asm ][ asm ][ asm ] + // [ asm ][ asm ][ asm ] + // [ m_left ] + // ------------------------------------------------------------------------- + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + mov ( imm ( 7 ), esi ) + kmovw ( esi, K ( 2 ) ) + + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm29 ) // xmm29 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- beta + + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + mov ( var ( m_iter ), r11 ) // ii = m_iter; + label ( .SLOOP3X4I ) // Outer loop over 6-row blocks of C/A + + mov ( r12, rcx ) // rcx = base of the current 6x3 output tile in C + + // Prefetch the rows of C that will be written to in this loop + mov ( var ( rs_c ), rsi ) // rsi = rs_c + lea ( mem ( , rsi, 4 ), rsi ) // rsi = rs_c * sizeof ( float ) + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, rsi, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, rsi, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, rsi, 4 ) ) // C row 4 + lea ( mem ( rcx, rsi, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, rsi, 1 ) ) // C row 3 = rcx + 3 * rs_c + lea ( mem ( rcx, rsi, 4 ), rax ) // rax = rcx + 4 * rs_c + prefetchw0 ( mem ( rax, rsi, 1 ) ) // C row 5 = rcx + 5 * rs_c + + mov ( r14, rax ) // restart A at the top of this row-block + mov ( rdx, rbx ) // load b to rbx + + lea ( mem ( r10, r8, 2 ), rdi ) // rdi = 5 * rs_a ( rdi is reused by the store path, so recompute each iteration ) + + // zmm8-zmm28 accumulate a 6x3 tile. + INIT_ACCUM_3COL + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 3 columns of B and update the 6x3 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 3 columns of B and update the 6x3 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // The previous k-loop decomposition used iterations of 64, 32, and 8 elements, which is inefficient for k values below 32. + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for 1-15 remaining k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), ZMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 17 MASK_KZ ( 1 ) ) ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 18 MASK_KZ ( 1 ) ) ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // For tails of <= 8 elements, use masked YMM operations for the + // remainder instead of looping element-by-element and issuing + // per-element FMAs, which is wasteful for this kernel. + // Perform a masked FMA operation on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), YMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 17 MASK_KZ ( 1 ) ) ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 18 MASK_KZ ( 1 ) ) ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm30 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 6x3 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 6x3 tile. + // ZMM_REDUCE_3 folds 3 such accumulators across k and packs the 3 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..2. + // C_STOR_MASKED then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), *] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), *] + ZMM_REDUCE_3 ( 10, 13, 16, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), *] + + ALPHA_SCALE ( 29, 4, 5, 6 ) // scale the first 3 rows by alpha + + mov ( var ( rs_c ), rdi ) // rdi = rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rdi = rs_c * sizeof ( float ) + C_STOR_MASKED ( rdi, 30, 4, 5, 6 ) // update the first 3 rows of the 6x3 tile + + ZMM_REDUCE_3 ( 20, 23, 26, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), *] + ZMM_REDUCE_3 ( 21, 24, 27, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), *] + ZMM_REDUCE_3 ( 22, 25, 28, 19 ) // xmm19 = [sum ( zmm22 ), sum ( zmm25 ), sum ( zmm28 ), *] + + ALPHA_SCALE ( 29, 17, 18, 19 ) // scale the next 3 rows by alpha + + C_STOR_MASKED_CONT ( rdi, 30, 17, 18, 19 ) // update the next 3 rows + + jmp ( .SDONE ) + + // Reduce the 6x3 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), *] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), *] + ZMM_REDUCE_3 ( 10, 13, 16, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), *] + + ALPHA_SCALE ( 29, 4, 5, 6 ) // scale the first 3 rows by alpha + + mov ( var ( rs_c ), rdi ) // rdi = rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rdi = rs_c * sizeof ( float ) + C_STOR_BZ_MASKED ( rdi, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_3 ( 20, 23, 26, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), *] + ZMM_REDUCE_3 ( 21, 24, 27, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), *] + ZMM_REDUCE_3 ( 22, 25, 28, 19 ) // xmm19 = [sum ( zmm22 ), sum ( zmm25 ), sum ( zmm28 ), *] + + ALPHA_SCALE ( 29, 17, 18, 19 ) // scale the next 3 rows by alpha + + C_STOR_BZ_MASKED_CONT ( rdi, 17, 18, 19 ) // store the next 3 rows + + label ( .SDONE ) + + mov ( var ( rs_c ), rdi ) // load rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rs_c *= sizeof ( float ) => rs_c *= 4 + lea ( mem ( r12, rdi, 2 ), r12 ) + lea ( mem ( r12, rdi, 4 ), r12 ) // c_ii = r12 += 6*rs_c + + lea ( mem ( r14, r8, 2 ), r14 ) + lea ( mem ( r14, r8, 4 ), r14 ) // a_ii = r14 += 6*rs_a + + dec ( r11 ) + jne ( .SLOOP3X4I ) // iterate again if ii != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [m_iter] "m" ( m_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1", "k2" + ) + + consider_edge_cases: + + // Control reaches here in two cases: + // 1. asm finished the full 6x3 tile work and only bottom-edge rows remain. + // 2. asm was skipped because m0 < 6. + // m_left dispatch: handle the bottom edge rows that remain after the asm + // kernel covers all full 6-row blocks. + // Computes the m_left part of C. + // [ - ][ - ][ - ] + // [ - ][ - ][ - ] + // [ - ][ - ][ - ] + // [ - ][ - ][ - ] + // [ - ][ - ][ - ] + // [ - ][ - ][ - ] + // [ m_left area ] + if ( m_left ) + { + const dim_t nr_cur = n0; + const dim_t i_edge = m0 - ( dim_t )m_left; + + // Start at row 6 * m_iter, i.e. the first row not covered by asm. + float* restrict cij = c + i_edge*rs_c; + float* restrict bj = b; + float* restrict ai = a + i_edge*rs_a; + + if ( 5 == m_left ) + { + dim_t mr_cur = 5; + bli_sgemmsup_rd_zen5_asm_5x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 4 == m_left ) + { + const dim_t mr_cur = 4; + + bli_sgemmsup_rd_zen5_asm_4x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 3 == m_left ) + { + const dim_t mr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_3x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == m_left ) + { + const dim_t mr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_2x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == m_left ) + { + const dim_t mr_cur = 1; + + bli_sgemmsup_rd_zen5_asm_1x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + } +} + +void bli_sgemmsup_rd_zen5_asm_6x2m + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 6 rows and 2 columns of C. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 6 rows at a time; 1-5 rows fall through to m_left dispatch. + uint64_t m_iter = m0 / 6; + uint64_t m_left = m0 % 6; + + uint64_t rs_a = rs_a0; + + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // The asm microkernel only handles full 6x2 tiles. + // If m0 < 6, skip straight to the C dispatch below. + if ( m_iter == 0 ) goto consider_edge_cases; + + // Main 6x2 microkernel over the full 6x2 rectangle: + // Handles all rows in multiples of 6. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ] + // [ asm ][ asm ] + // [ asm ][ asm ] + // [ asm ][ asm ] + // [ asm ][ asm ] + // [ asm ][ asm ] + // [ m_left ] + // ------------------------------------------------------------------------- + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm14 ) // xmm14 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm15 ) // xmm15 <- beta + + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + mov ( var ( m_iter ), r11 ) // ii = m_iter; + label ( .SLOOP3X4I ) // Outer loop over 6-row blocks of C/A + + mov ( r12, rcx ) // rcx = base of the current 6x2 output tile in C + + // Prefetch the rows of C that will be written to in this loop + mov ( var ( rs_c ), rsi ) // rsi = rs_c + lea ( mem ( , rsi, 4 ), rsi ) // rsi = rs_c * sizeof ( float ) + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, rsi, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, rsi, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, rsi, 4 ) ) // C row 4 + lea ( mem ( rcx, rsi, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, rsi, 1 ) ) // C row 3 = rcx + 3 * rs_c + lea ( mem ( rcx, rsi, 4 ), rax ) // rax = rcx + 4 * rs_c + prefetchw0 ( mem ( rax, rsi, 1 ) ) // C row 5 = rcx + 5 * rs_c + + mov ( r14, rax ) // restart A at the top of this row-block + mov ( rdx, rbx ) // load b to rbx + + lea ( mem ( r10, r8, 2 ), rdi ) // rdi = 5 * rs_a ( rdi is reused by the store path, so recompute each iteration ) + + // zmm8-zmm25 accumulate a 6x2 tile. + INIT_ACCUM_2COL + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 2 columns of B and update the 6x2 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 2 columns of B and update the 6x2 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // The previous k-loop decomposition used iterations of 64, 32, and 8 elements, which is inefficient for k values below 32. + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), ZMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 17 MASK_KZ ( 1 ) ) ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // For tails of <= 8 elements, use masked YMM operations for the + // remainder instead of looping element-by-element and issuing + // per-element FMAs, which is wasteful for this kernel. + // Perform a masked FMA operation on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), YMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 17 MASK_KZ ( 1 ) ) ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm15 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 6x2 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 6x2 tile. + // ZMM_REDUCE_2 folds 2 such accumulators across k and packs the 2 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..1. + // C_STOR_2_FLOATS then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), *, *] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), *, *] + ZMM_REDUCE_2 ( 10, 13, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), *, *] + + ALPHA_SCALE ( 14, 4, 5, 6 ) // scale the first 3 rows by alpha + + mov ( var ( rs_c ), rdi ) // rdi = rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rdi = rs_c * sizeof ( float ) + C_STOR_2_FLOATS ( rdi, 15, 4, 5, 6 ) // update the first 3 rows of the 6x2 tile + + ZMM_REDUCE_2 ( 20, 23, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 ), *, *] + ZMM_REDUCE_2 ( 21, 24, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 ), *, *] + ZMM_REDUCE_2 ( 22, 25, 19 ) // xmm19 = [sum ( zmm22 ), sum ( zmm25 ), *, *] + + ALPHA_SCALE ( 14, 17, 18, 19 ) // scale the next 3 rows by alpha + + C_STOR_2_FLOATS_CONT ( rdi, 15, 17, 18, 19 ) // update the next 3 rows + + jmp ( .SDONE ) + + // Reduce the 6x2 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), *, *] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), *, *] + ZMM_REDUCE_2 ( 10, 13, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), *, *] + + ALPHA_SCALE ( 14, 4, 5, 6 ) // scale the first 3 rows by alpha + + mov ( var ( rs_c ), rdi ) // rdi = rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rdi = rs_c * sizeof ( float ) + C_STOR_BZ_2_FLOATS ( rdi, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_2 ( 20, 23, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 ), *, *] + ZMM_REDUCE_2 ( 21, 24, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 ), *, *] + ZMM_REDUCE_2 ( 22, 25, 19 ) // xmm19 = [sum ( zmm22 ), sum ( zmm25 ), *, *] + + ALPHA_SCALE ( 14, 17, 18, 19 ) // scale the next 3 rows by alpha + + C_STOR_BZ_2_FLOATS_CONT ( rdi, 17, 18, 19 ) // store the next 3 rows + + label ( .SDONE ) + + mov ( var ( rs_c ), rdi ) // load rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rs_c *= sizeof ( float ) => rs_c *= 4 + lea ( mem ( r12, rdi, 2 ), r12 ) + lea ( mem ( r12, rdi, 4 ), r12 ) // c_ii = r12 += 6*rs_c + + lea ( mem ( r14, r8, 2 ), r14 ) + lea ( mem ( r14, r8, 4 ), r14 ) // a_ii = r14 += 6*rs_a + + dec ( r11 ) + jne ( .SLOOP3X4I ) // iterate again if ii != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [m_iter] "m" ( m_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) + + consider_edge_cases: + + // Control reaches here in two cases: + // 1. asm finished the full 6x2 tile work and only bottom-edge rows remain. + // 2. asm was skipped because m0 < 6. + // m_left dispatch: handle the bottom edge rows that remain after the asm + // kernel covers all full 6-row blocks. + // Computes the m_left part of C. + // [ - ][ - ] + // [ - ][ - ] + // [ - ][ - ] + // [ - ][ - ] + // [ - ][ - ] + // [ - ][ - ] + // [ m_left ] + if ( m_left ) + { + const dim_t nr_cur = n0; + const dim_t i_edge = m0 - ( dim_t )m_left; + + // Start at row 6 * m_iter, i.e. the first row not covered by asm. + float* restrict cij = c + i_edge*rs_c; + float* restrict bj = b; + float* restrict ai = a + i_edge*rs_a; + + if ( 5 == m_left ) + { + dim_t mr_cur = 5; + bli_sgemmsup_rd_zen5_asm_5x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 4 == m_left ) + { + const dim_t mr_cur = 4; + + bli_sgemmsup_rd_zen5_asm_4x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 3 == m_left ) + { + const dim_t mr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_3x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == m_left ) + { + const dim_t mr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_2x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == m_left ) + { + const dim_t mr_cur = 1; + + bli_sgemmsup_rd_zen5_asm_1x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + } +} diff --git a/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64n.c b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64n.c new file mode 100644 index 000000000..155795ec0 --- /dev/null +++ b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64n.c @@ -0,0 +1,2456 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +#define BLIS_ASM_SYNTAX_ATT +#include "bli_x86_asm_macros.h" + +#include "bli_gemmsup_rd_zen5_asm_s6x64.h" + +void bli_sgemmsup_rd_zen5_asm_6x64n + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This function handles m0 <= 6 rows of C. + // The asm path below is used only when m0 == 6. + uint64_t m_left = m0 % 6; + + // For m0 < 6, decompose the row edge into 3x64n, 2x64n, and finally gemv + // for the last row. + if ( m_left ) + { + float* restrict cij = c; + float* restrict bj = b; + float* restrict ai = a; + + if ( 5 <= m_left ) + { + const dim_t mr_cur = 5; + + bli_sgemmsup_rd_zen5_asm_5x64n + ( + conja, conjb, mr_cur, n0, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 4 <= m_left ) + { + const dim_t mr_cur = 4; + + bli_sgemmsup_rd_zen5_asm_4x64n + ( + conja, conjb, mr_cur, n0, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 3 <= m_left ) + { + const dim_t mr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_3x64n + ( + conja, conjb, mr_cur, n0, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 <= m_left ) + { + const dim_t mr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_2x64n + ( + conja, conjb, mr_cur, n0, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == m_left ) + { + bli_sgemv_ex + ( + BLIS_TRANSPOSE, conja, k0, n0, + alpha, bj, rs_b0, cs_b0, ai, cs_a0, + beta, cij, cs_c0, cntx, NULL + ); + } + return; + } + + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns at a time; 1-3 columns fall through to n_left dispatch. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + uint64_t cs_c = cs_c0; + + // Control reaches here when the main 4-column asm loop is skipped or when + // only the right-edge columns remain. + // n_left dispatch: handle the right edge columns that remain after the asm + // kernel covers all full 4-column blocks. + // Computes the n_left part of C. + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + if ( n_left ) + { + const dim_t mr_cur = 6; + const dim_t j_edge = n0 - ( dim_t )n_left; + + // Start at column 4 * n_iter, i.e. the first column not covered by asm. + float* restrict cij = c + j_edge*cs_c; + float* restrict ai = a; + float* restrict bj = b + j_edge*cs_b; + + if ( 3 == n_left ) + { + const dim_t nr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_6x3m + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == n_left ) + { + const dim_t nr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_6x2m + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == n_left ) + { + bli_sgemv_ex + ( + BLIS_NO_TRANSPOSE, conjb, mr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, + beta, cij, rs_c0, cntx, NULL + ); + } + } + + // The asm microkernel only handles a single full 6x4 tile in the m dimension. + if ( n_iter == 0 ) return; + + float *abuf = a; + float *bbuf = b + ( n_iter - 1 ) * 4 * cs_b; + float *cbuf = c + ( n_iter - 1 ) * 4 * cs_c; + + // Main 6x4 microkernel over the full top-left rectangle: + // Handles exactly 6 rows and columns in multiples of 4. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // ------------------------------------------------------------------------- + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + label ( .SLOOP3X4I ) // Single 6-row block + + mov ( var ( abuf ), rdx ) // load base address of a + mov ( var ( bbuf ), r14 ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + mov ( var ( n_iter ), r15 ) // jj = n_iter; + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + lea ( mem ( r10, r8, 2 ), rdi ) // rdi = 5 * rs_a + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( r12, rcx ) // rcx = base of the current 6x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, r11, 4 ) ) // C row 4 + lea ( mem ( rcx, r11, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, r11, 1 ) ) // C row 3 + lea ( mem ( rcx, r11, 4 ), rax ) // rax = rcx + 4 * rs_c + prefetchw0 ( mem ( rax, r11, 1 ) ) // C row 5 + mov ( rdx, rax ) // restart A at the top of this 4-col block + mov ( r14, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm31 accumulate a 6x4 tile. + INIT_ACCUM_4COL + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 6x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 6x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 6x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), ZMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), YMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + label ( .POST_ACCUM ) + + // Preload alpha into xmm7 to avoid redundant memory reloads. + mov ( var ( alpha ), rax ) // load address of alpha + vbroadcastss ( ( rax ), xmm7 ) // xmm7 = alpha + mov ( var ( beta ), rax ) // load address of beta + vbroadcastss ( ( rax ), xmm0 ) + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm0 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 6x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 6x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + // C_STOR then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 7, 4, 5, 6 ) // scale the first 3 rows by alpha + + // Preload beta into xmm9 now that zmm9 is free ( consumed by ZMM_REDUCE_4 ). + mov ( var ( beta ), rax ) + vbroadcastss ( ( rax ), xmm9 ) // xmm9 = beta + + C_STOR ( r11, 9, 4, 5, 6 ) // update the first 3 rows of the 6x4 tile + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + ZMM_REDUCE_4 ( 21, 24, 27, 30, 11 ) // xmm11 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), sum ( zmm30 )] + ZMM_REDUCE_4 ( 22, 25, 28, 31, 14 ) // xmm14 = [sum ( zmm22 ), sum ( zmm25 ), sum ( zmm28 ), sum ( zmm31 )] + + ALPHA_SCALE ( 7, 8, 11, 14 ) // scale the next 3 rows by alpha + + C_STOR_CONT ( r11, 9, 8, 11, 14 ) // update the next 3 rows + + jmp ( .SDONE ) + + // Reduce the 6x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 7, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + ZMM_REDUCE_4 ( 21, 24, 27, 30, 11 ) // xmm11 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), sum ( zmm30 )] + ZMM_REDUCE_4 ( 22, 25, 28, 31, 14 ) // xmm14 = [sum ( zmm22 ), sum ( zmm25 ), sum ( zmm28 ), sum ( zmm31 )] + + ALPHA_SCALE ( 7, 8, 11, 14 ) // scale the next 3 rows by alpha + + C_STOR_BZ_CONT ( r11, 8, 11, 14 ) // store the next 3 rows + + label ( .SDONE ) + + sub ( imm ( 4*4 ), r12 ) // r12 -= 4 columns + lea ( mem ( , r9, 4 ), rsi ) // rsi = 4 * cs_b_bytes + sub ( rsi, r14 ) // r14 -= 4 * cs_b_bytes + + dec ( r15 ) + jne ( .SLOOP3X4J ) // iterate again if ii != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_iter] "m" ( n_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_5x64n + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 5 rows of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns at a time; 1-3 columns fall through to n_left dispatch. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + uint64_t cs_c = cs_c0; + + // Control reaches here when the main 4-column asm loop is skipped or when + // only the right-edge columns remain. + // n_left dispatch: handle the right edge columns that remain after the asm + // kernel covers all full 4-column blocks. + // Computes the n_left part of C. + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + if ( n_left ) + { + const dim_t mr_cur = 5; + const dim_t j_edge = n0 - ( dim_t )n_left; + + // Start at column 4 * n_iter, i.e. the first column not covered by asm. + float* restrict cij = c + j_edge*cs_c; + float* restrict ai = a; + float* restrict bj = b + j_edge*cs_b; + + if ( 3 == n_left ) + { + const dim_t nr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_5x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == n_left ) + { + const dim_t nr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_5x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == n_left ) + { + bli_sgemv_ex + ( + BLIS_NO_TRANSPOSE, conjb, mr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, + beta, cij, rs_c0, cntx, NULL + ); + } + } + + // The asm microkernel only handles full 5x4 tiles. + if ( n_iter == 0 ) return; + + float *abuf = a; + float *bbuf = b + ( n_iter - 1 ) * 4 * cs_b; + float *cbuf = c + ( n_iter - 1 ) * 4 * cs_c; + + // Main 5x4 microkernel over the full top-left rectangle: + // Handles 5 rows and columns in multiples of 4. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rdx ) // load base address of a + mov ( var ( bbuf ), r14 ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + mov ( var ( n_iter ), r15 ) // jj = n_iter; + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( r12, rcx ) // rcx = base of the current 5x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, r11, 4 ) ) // C row 4 + lea ( mem ( rcx, r11, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, r11, 1 ) ) // C row 3 + mov ( rdx, rax ) // restart A at the top of this 4-col block + mov ( r14, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm19 accumulate a 5x4 tile. + INIT_ACCUM_5x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + label ( .POST_ACCUM ) + + // Preload alpha into xmm7 to avoid redundant memory reloads. + mov ( var ( alpha ), rax ) // load address of alpha + vbroadcastss ( ( rax ), xmm7 ) // xmm7 = alpha + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 3x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 3x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + // C_STOR then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 7, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR ( r11, 31, 4, 5, 6 ) // update the 3x4 tile in C + + ZMM_REDUCE_4 ( 20, 21, 23, 24, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm21 ), sum ( zmm23 ), sum ( zmm24 )] + ZMM_REDUCE_4 ( 26, 27, 29, 30, 11 ) // xmm11 = [sum ( zmm26 ), sum ( zmm27 ), sum ( zmm29 ), sum ( zmm30 )] + + ALPHA_SCALE2 ( 7, 8, 11 ) // scale the next 2 rows by alpha + + C_STOR2_CONT ( r11, 31, 8, 11 ) // update the next 2 rows + + jmp ( .SDONE ) + + // Reduce the 3x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 7, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the 3x4 tile without reading C + + ZMM_REDUCE_4 ( 20, 21, 23, 24, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm21 ), sum ( zmm23 ), sum ( zmm24 )] + ZMM_REDUCE_4 ( 26, 27, 29, 30, 11 ) // xmm11 = [sum ( zmm26 ), sum ( zmm27 ), sum ( zmm29 ), sum ( zmm30 )] + + ALPHA_SCALE2 ( 7, 8, 11 ) // scale the next 2 rows by alpha + + C_STOR_BZ2_CONT ( r11, 8, 11 ) // store the next 2 rows + + label ( .SDONE ) + + sub ( imm ( 4*4 ), r12 ) // r12 -= 4 columns + + lea ( mem ( , r9, 4 ), rsi ) // rsi = 4 * cs_b_bytes + sub ( rsi, r14 ) // r14 -= 4 * cs_b_bytes + + dec ( r15 ) + jne ( .SLOOP3X4J ) // iterate again if jj != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_iter] "m" ( n_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_4x64n + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 4 rows of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns at a time; 1-3 columns fall through to n_left dispatch. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + + uint64_t rs_a = rs_a0; + + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + uint64_t cs_c = cs_c0; + + // Control reaches here when the main 4-column asm loop is skipped or when + // only the right-edge columns remain. + // n_left dispatch: handle the right edge columns that remain after the asm + // kernel covers all full 4-column blocks. + // Computes the n_left part of C. + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + if ( n_left ) + { + const dim_t mr_cur = 4; + const dim_t j_edge = n0 - ( dim_t )n_left; + + // Start at column 4 * n_iter, i.e. the first column not covered by asm. + float* restrict cij = c + j_edge*cs_c; + float* restrict ai = a; + float* restrict bj = b + j_edge*cs_b; + + if ( 3 == n_left ) + { + const dim_t nr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_4x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == n_left ) + { + const dim_t nr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_4x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == n_left ) + { + bli_sgemv_ex + ( + BLIS_NO_TRANSPOSE, conjb, mr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, + beta, cij, rs_c0, cntx, NULL + ); + } + } + + // The asm microkernel only handles full 4x4 tiles. + if ( n_iter == 0 ) return; + + float *abuf = a; + float *bbuf = b + ( n_iter - 1 ) * 4 * cs_b; + float *cbuf = c + ( n_iter - 1 ) * 4 * cs_c; + + // Main 4x4 microkernel over the full top-left rectangle: + // Handles 4 rows and columns in multiples of 4. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rdx ) // load base address of a + mov ( var ( bbuf ), r14 ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + mov ( var ( n_iter ), r15 ) // jj = n_iter; + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( r12, rcx ) // rcx = base of the current 4x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + lea ( mem ( rcx, r11, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, r11, 1 ) ) // C row 3 + mov ( rdx, rax ) // restart A at the top of this 4-col block + mov ( r14, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm20 accumulate a 4x4 tile. + INIT_ACCUM_4x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 4 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 4x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 21 MASK_KZ ( 1 ) ) ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + // Perform a masked FMA operation on the remaining elements + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 21 MASK_KZ ( 1 ) ) ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 3x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 3x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + // C_STOR then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR ( r11, 31, 4, 5, 6 ) // update the 3x4 tile in C + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + + ALPHA_SCALE1 ( 30, 8 ) // scale the 4 elements by alpha + + C_STOR1_CONT ( r11, 31, 8 ) // update the next row + + jmp ( .SDONE ) + + // Reduce the 3x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the 3x4 tile without reading C + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + + ALPHA_SCALE1 ( 30, 8 ) // scale the 4 elements by alpha + + C_STOR_BZ1_CONT ( r11, 8 ) // store the next row + + label ( .SDONE ) + + sub ( imm ( 4*4 ), r12 ) // r12 -= 4 columns + + lea ( mem ( , r9, 4 ), rsi ) // rsi = 4 * cs_b_bytes + sub ( rsi, r14 ) // r14 -= 4 * cs_b_bytes + + dec ( r15 ) + jne ( .SLOOP3X4J ) // iterate again if jj != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_iter] "m" ( n_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_3x64n + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 3 rows of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns at a time; 1-3 columns fall through to n_left dispatch. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + + uint64_t rs_a = rs_a0; + + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + uint64_t cs_c = cs_c0; + + // Control reaches here when the main 4-column asm loop is skipped or when + // only the right-edge columns remain. + // n_left dispatch: handle the right edge columns that remain after the asm + // kernel covers all full 4-column blocks. + // Computes the n_left part of C. + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + if ( n_left ) + { + const dim_t mr_cur = 3; + const dim_t j_edge = n0 - ( dim_t )n_left; + + // Start at column 4 * n_iter, i.e. the first column not covered by asm. + float* restrict cij = c + j_edge*cs_c; + float* restrict ai = a; + float* restrict bj = b + j_edge*cs_b; + + if ( 3 == n_left ) + { + const dim_t nr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_3x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == n_left ) + { + const dim_t nr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_3x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == n_left ) + { + bli_sgemv_ex + ( + BLIS_NO_TRANSPOSE, conjb, mr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, + beta, cij, rs_c0, cntx, NULL + ); + } + } + + // The asm microkernel only handles full 3x4 tiles. + if ( n_iter == 0 ) return; + + float *abuf = a; + float *bbuf = b + ( n_iter - 1 ) * 4 * cs_b; + float *cbuf = c + ( n_iter - 1 ) * 4 * cs_c; + + // Main 3x4 microkernel over the full top-left rectangle: + // Handles 3 rows and columns in multiples of 4. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rdx ) // load base address of a + mov ( var ( bbuf ), r14 ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + mov ( var ( n_iter ), r15 ) // jj = n_iter; + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( r12, rcx ) // rcx = base of the current 3x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + mov ( rdx, rax ) // restart A at the top of this 4-col block + mov ( r14, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm19 accumulate a 3x4 tile. + INIT_ACCUM_3x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 20 MASK_KZ ( 1 ) ) ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 21 MASK_KZ ( 1 ) ) ) + VFMA3 ( 21, 17, 18, 19 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 20 MASK_KZ ( 1 ) ) ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 21 MASK_KZ ( 1 ) ) ) + VFMA3 ( 21, 17, 18, 19 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 3x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 3x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + // C_STOR then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR ( r11, 31, 4, 5, 6 ) // update the 3x4 tile in C + + jmp ( .SDONE ) + + // Reduce the 3x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the 3x4 tile without reading C + + label ( .SDONE ) + + sub ( imm ( 4*4 ), r12 ) // r12 -= 4 columns + + lea ( mem ( , r9, 4 ), rsi ) // rsi = 4 * cs_b_bytes + sub ( rsi, r14 ) // r14 -= 4 * cs_b_bytes + + dec ( r15 ) + jne ( .SLOOP3X4J ) // iterate again if jj != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_iter] "m" ( n_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_2x64n + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 2 rows of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns at a time; 1-3 columns fall through to n_left dispatch. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + + uint64_t rs_a = rs_a0; + + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + uint64_t cs_c = cs_c0; + + // Control reaches here when the main 4-column asm loop is skipped or when + // only the right-edge columns remain. + // n_left dispatch: handle the right edge columns that remain after the asm + // kernel covers all full 4-column blocks. + // Computes the n_left part of C. + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + if ( n_left ) + { + const dim_t mr_cur = 2; + const dim_t j_edge = n0 - ( dim_t )n_left; + + // Start at column 4 * n_iter, i.e. the first column not covered by asm. + float* restrict cij = c + j_edge*cs_c; + float* restrict ai = a; + float* restrict bj = b + j_edge*cs_b; + + if ( 3 == n_left ) + { + const dim_t nr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_2x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == n_left ) + { + const dim_t nr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_2x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == n_left ) + { + bli_sgemv_ex + ( + BLIS_NO_TRANSPOSE, conjb, mr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, + beta, cij, rs_c0, cntx, NULL + ); + } + } + + // The asm microkernel only handles full 2x4 tiles. + if ( n_iter == 0 ) return; + + float *abuf = a; + float *bbuf = b + ( n_iter - 1 ) * 4 * cs_b; + float *cbuf = c + ( n_iter - 1 ) * 4 * cs_c; + + // Main 2x4 microkernel over the full top-left rectangle: + // Handles 2 rows and columns in multiples of 4. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rdx ) // load base address of a + mov ( var ( bbuf ), r14 ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + mov ( var ( n_iter ), r15 ) // jj = n_iter; + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( r12, rcx ) // rcx = base of the current 2x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + mov ( rdx, rax ) // restart A at the top of this 4-col block + mov ( r14, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm18 accumulate a 2x4 tile. + INIT_ACCUM_2x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 2x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 2x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 2x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 20 MASK_KZ ( 1 ) ) ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 21 MASK_KZ ( 1 ) ) ) + VFMA2 ( 21, 17, 18 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 20 MASK_KZ ( 1 ) ) ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 21 MASK_KZ ( 1 ) ) ) + VFMA2 ( 21, 17, 18 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 2x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 2x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5 <- rows 0..1, cols 0..3. + // C_STOR2 then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR2 ( r11, 31, 4, 5 ) // update the 2x4 tile in C + + jmp ( .SDONE ) + + // Reduce the 2x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR_BZ2 ( r11, 4, 5 ) // store the 2x4 tile without reading C + + label ( .SDONE ) + + sub ( imm ( 4*4 ), r12 ) // r12 -= 4 columns + lea ( mem ( , r9, 4 ), rsi ) // rsi = 4 * cs_b_bytes + sub ( rsi, r14 ) // r14 -= 4 * cs_b_bytes + + dec ( r15 ) + jne ( .SLOOP3X4J ) // iterate again if jj != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_iter] "m" ( n_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm17", "ymm18", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} diff --git a/kernels/zen5/bli_kernels_zen5.h b/kernels/zen5/bli_kernels_zen5.h index bd0445014..49d1b8110 100644 --- a/kernels/zen5/bli_kernels_zen5.h +++ b/kernels/zen5/bli_kernels_zen5.h @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2024 - 2025, Advanced Micro Devices, Inc. All rights reserved. + Copyright (C) 2024 - 2026, Advanced Micro Devices, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -32,6 +32,8 @@ */ +#include "bli_gemm_tiny_zen5.h" + // DCOPYV kernel COPYV_KER_PROT( double, d, copyv_zen5_asm ) @@ -45,6 +47,32 @@ GEMMSUP_KER_PROT( double, d, gemmsup_cv_zen5_asm_24x3m) GEMMSUP_KER_PROT( double, d, gemmsup_cv_zen5_asm_24x2m) GEMMSUP_KER_PROT( double, d, gemmsup_cv_zen5_asm_24x1m) + +// sgemm sup RD kernels +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_6x64m ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_6x3m ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_6x2m ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_5x64 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_4x64 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_3x64 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_2x64 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_1x64 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_5x3 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_4x3 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_3x3 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_2x3 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_1x3 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_5x2 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_4x2 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_3x2 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_2x2 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_1x2 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_6x64n ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_5x64n ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_4x64n ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_3x64n ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_2x64n ) + // threshold functions bool bli_cntx_gemmsup_thresh_is_met_zen5 (