diff --git a/config/zen5/bli_cntx_init_zen5.c b/config/zen5/bli_cntx_init_zen5.c index 1e5a009ea..fdf730024 100644 --- a/config/zen5/bli_cntx_init_zen5.c +++ b/config/zen5/bli_cntx_init_zen5.c @@ -314,11 +314,11 @@ void bli_cntx_init_zen5( cntx_t* cntx ) BLIS_CCC, BLIS_DOUBLE, bli_dgemmsup_cv_zen5_asm_24x8m, FALSE, BLIS_RRR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64m, TRUE, - BLIS_RRC, BLIS_FLOAT, bli_sgemmsup_rd_zen4_asm_6x64m, TRUE, + BLIS_RRC, BLIS_FLOAT, bli_sgemmsup_rd_zen5_asm_6x64m, TRUE, BLIS_RCR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64m, TRUE, BLIS_RCC, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64n, TRUE, BLIS_CRR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64m, TRUE, - BLIS_CRC, BLIS_FLOAT, bli_sgemmsup_rd_zen4_asm_6x64n, TRUE, + BLIS_CRC, BLIS_FLOAT, bli_sgemmsup_rd_zen5_asm_6x64n, TRUE, BLIS_CCR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64n, TRUE, BLIS_CCC, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64n, TRUE, diff --git a/config/zen5/bli_config_zen5.h b/config/zen5/bli_config_zen5.h index bf0f91671..91f046ea6 100644 --- a/config/zen5/bli_config_zen5.h +++ b/config/zen5/bli_config_zen5.h @@ -171,7 +171,7 @@ if ( PASTECH2( THRESH_GEMM_, ch, _TINY_ZEN5_AVX2 )( transa, transb, m, n, k, is_parallel ) ) \ LOOKUP_AVX2_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) \ else if ( PASTECH2( THRESH_GEMM_, ch, _TINY_ZEN5_AVX512 )( stor_id, transa, transb, m, n, k, is_parallel ) ) \ - LOOKUP_AVX512_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) \ + LOOKUP_ZEN5_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) \ break; #endif diff --git a/config/zen6/bli_cntx_init_zen6.c b/config/zen6/bli_cntx_init_zen6.c index d7a3d36e6..341c6c6bf 100644 --- a/config/zen6/bli_cntx_init_zen6.c +++ b/config/zen6/bli_cntx_init_zen6.c @@ -314,11 +314,11 @@ void bli_cntx_init_zen6( cntx_t* cntx ) BLIS_CCC, BLIS_DOUBLE, bli_dgemmsup_cv_zen5_asm_24x8m, FALSE, BLIS_RRR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64m, TRUE, - BLIS_RRC, BLIS_FLOAT, bli_sgemmsup_rd_zen4_asm_6x64m, TRUE, + BLIS_RRC, BLIS_FLOAT, bli_sgemmsup_rd_zen5_asm_6x64m, TRUE, BLIS_RCR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64m, TRUE, BLIS_RCC, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64n, TRUE, BLIS_CRR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64m, TRUE, - BLIS_CRC, BLIS_FLOAT, bli_sgemmsup_rd_zen4_asm_6x64n, TRUE, + BLIS_CRC, BLIS_FLOAT, bli_sgemmsup_rd_zen5_asm_6x64n, TRUE, BLIS_CCR, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64n, TRUE, BLIS_CCC, BLIS_FLOAT, bli_sgemmsup_rv_zen4_asm_6x64n, TRUE, diff --git a/config/zen6/bli_config_zen6.h b/config/zen6/bli_config_zen6.h index c85ead46d..c46f79665 100644 --- a/config/zen6/bli_config_zen6.h +++ b/config/zen6/bli_config_zen6.h @@ -171,7 +171,7 @@ if ( PASTECH2( THRESH_GEMM_, ch, _TINY_ZEN6_AVX2 )( transa, transb, m, n, k, is_parallel ) ) \ LOOKUP_AVX2_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) \ else if ( PASTECH2( THRESH_GEMM_, ch, _TINY_ZEN6_AVX512 )( stor_id, transa, transb, m, n, k, is_parallel ) ) \ - LOOKUP_AVX512_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) \ + LOOKUP_ZEN5_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) /* Using the zen5 configuration */ \ break; #endif diff --git a/gtestsuite/testinghelpers/inc/common/blis_version_defs.h b/gtestsuite/testinghelpers/inc/common/blis_version_defs.h index 68225c28c..e15b1a5b2 100644 --- a/gtestsuite/testinghelpers/inc/common/blis_version_defs.h +++ b/gtestsuite/testinghelpers/inc/common/blis_version_defs.h @@ -125,6 +125,8 @@ #define K_bli_dnorm2fv_zen_int_unb_var1 bli_dnorm2fv_zen_int_unb_var1 #define K_bli_sgemmsup_rd_zen4_asm_6x64n bli_sgemmsup_rd_zen4_asm_6x64n #define K_bli_sgemmsup_rd_zen4_asm_6x64m bli_sgemmsup_rd_zen4_asm_6x64m + #define K_bli_sgemmsup_rd_zen5_asm_6x64n bli_sgemmsup_rd_zen5_asm_6x64n + #define K_bli_sgemmsup_rd_zen5_asm_6x64m bli_sgemmsup_rd_zen5_asm_6x64m #define K_bli_sgemmsup_rv_zen4_asm_6x64n bli_sgemmsup_rv_zen4_asm_6x64n #define K_bli_sgemmsup_rv_zen4_asm_6x64m bli_sgemmsup_rv_zen4_asm_6x64m #define K_bli_sgemmsup_rv_zen4_asm_6x64n bli_sgemmsup_rv_zen4_asm_6x64n diff --git a/gtestsuite/testsuite/ukr/gemm/sgemm/sgemm_ukernel.cpp b/gtestsuite/testsuite/ukr/gemm/sgemm/sgemm_ukernel.cpp index 9f869ab00..daadca44c 100644 --- a/gtestsuite/testsuite/ukr/gemm/sgemm/sgemm_ukernel.cpp +++ b/gtestsuite/testsuite/ukr/gemm/sgemm/sgemm_ukernel.cpp @@ -519,6 +519,136 @@ INSTANTIATE_TEST_SUITE_P( #endif // defined(BLIS_KERNELS_ZEN4) && defined(GTEST_AVX512) + +#if defined(BLIS_KERNELS_ZEN5) && defined(GTEST_AVX512) + +/* + The bli_sgemmsup_rd_zen5_asm_6x64m(standalone), accepts inputs with the + following contingency for n. + n <= NR, where NR is 64 + The code structure for the sgemm_sup rd kernels(m-var) are as follows: + In m direction : + Main kernel : Blocks of 6(L6_M) + Fringe kernels : 5 ... 1(L5_M ... L1_M) + In k direction : + Main loop : Blocks of 64(L64_K) + Fringe loop : Blocks of 32, 16, followed by a masked tail for 1..15 + In n direction : + Main kernel : NR = 64(L64_N) + Fringe kernels : With n being 3, 2, 1(AVX512 kernels)(L3_N, L2_N, L1_N) + + The inherent storage scheme format for the kernel is RRC, for C, A and B. + The testing interface allows for testing row-storage(inherent) and col-storage(operation transpose) + of C. We still need to pass the right transpose value pair for A and B, as per the kernel requirement. +*/ + +// Checking with row storage of C +#ifdef K_bli_sgemmsup_rd_zen5_asm_6x64m +INSTANTIATE_TEST_SUITE_P( + bli_sgemmsup_rd_zen5_asm_6x64m_row_stored_c, + sgemmGenericSUP, + ::testing::Combine( + ::testing::Range(gtint_t(1), gtint_t(7), gtint_t(1)), // values of m(L6_M to L1_M) + ::testing::Values(gtint_t(64), // values of n, L64_N + gtint_t(48), // L48_N + gtint_t(32), // L32_N + gtint_t(8), // L8_N + gtint_t(7), // L4_N + L3_N + gtint_t(3), // L3_N + gtint_t(2), // L2_N + gtint_t(1), // L1_N + gtint_t(63)), // Combination of fringe cases for N + ::testing::Values(gtint_t(64), // values of k, L64_K + gtint_t(32), // L32_K + gtint_t(16), // L16_K + gtint_t(12), // L_gt_8_K + gtint_t(7), // L_lt_8_K + gtint_t(256), // 4 * L64_K + gtint_t(303)), // Combination of main and fringe cases for K + ::testing::Values(2.0, 1.0, -1.0), // alpha value + ::testing::Values(1.0, 0.0, -1.0, 2.3), // beta value + ::testing::Values('r'), // storage of c + ::testing::Values(K_bli_sgemmsup_rd_zen5_asm_6x64m), // sgemm_sup_kernel + ::testing::Values(gtint_t(6)), // Micro kernel block MR + ::testing::Values('n'), // transa, has to be N for row storage + ::testing::Values('t'), // transb, has to be T for row storage + ::testing::Values(true), // kernel pref + ::testing::Values(true, false) // memory test + ), + ::sgemmGenericSUPPrint() + ); +#endif + +// Checking with col storage of C +// NOTE : Since we are inducing transpose at operation level, for code coverage, we +// have to interchange m and n instantiations +#ifdef K_bli_sgemmsup_rd_zen5_asm_6x64m +INSTANTIATE_TEST_SUITE_P( + bli_sgemmsup_rd_zen5_asm_6x64m_col_stored_c, + sgemmGenericSUP, + ::testing::Combine( + ::testing::Values(gtint_t(64), // values of n, L64_N + gtint_t(48), // L48_N + gtint_t(32), // L32_N + gtint_t(8), // L8_N + gtint_t(7), // L4_N + L3_N + gtint_t(3), // L3_N + gtint_t(2), // L2_N + gtint_t(1), // L1_N + gtint_t(63)), // Combination of fringe cases for N + ::testing::Range(gtint_t(1), gtint_t(7), gtint_t(1)), // values of n(L6_M to L1_M) + ::testing::Values(gtint_t(64), // values of k, L64_K + gtint_t(32), // L32_K + gtint_t(16), // L16_K + gtint_t(12), // L_gt_8_K + gtint_t(7), // L_lt_8_K + gtint_t(256), // 4 * L64_K + gtint_t(303)), // Combination of main and fringe cases for K + ::testing::Values(2.0, 1.0, -1.0), // alpha value + ::testing::Values(1.0, 0.0, -1.0, 2.3), // beta value + ::testing::Values('c'), // storage of c + ::testing::Values(K_bli_sgemmsup_rd_zen5_asm_6x64m), // sgemm_sup_kernel + ::testing::Values(gtint_t(6)), // Micro kernel block MR + ::testing::Values('t'), // transa, has to be T for row storage + ::testing::Values('n'), // transb, has to be N for row storage + ::testing::Values(true), // kernel pref + ::testing::Values(true, false) // memory test + ), + ::sgemmGenericSUPPrint() + ); +#endif + + +#ifdef K_bli_sgemmsup_rd_zen5_asm_6x64n +INSTANTIATE_TEST_SUITE_P( + bli_sgemmsup_rd_zen5_asm_6x64n_row_stored_c, + sgemmGenericSUP, + ::testing::Combine( + ::testing::Range(gtint_t(1), gtint_t(7), 1), // values of m + ::testing::Range(gtint_t(1), gtint_t(65), 1), // values of n + ::testing::Values(gtint_t(64), // values of k, L64_K + gtint_t(32), // L32_K + gtint_t(16), // L16_K + gtint_t(12), // L_gt_8_K + gtint_t(7), // L_lt_8_K + gtint_t(256), // 4 * L64_K + gtint_t(303)), // Combination of main and fringe cases for K + ::testing::Values(2.0, 1.0, -1.0), // alpha value + ::testing::Values(1.0, 0.0, -1.0, 2.3), // beta value + ::testing::Values('r'), // storage of c + ::testing::Values(K_bli_sgemmsup_rd_zen5_asm_6x64n), // sgemm_sup_kernel + ::testing::Values(gtint_t(6)), // Micro kernel block MR + ::testing::Values('n'), // transa + ::testing::Values('t'), // transb + ::testing::Values(false), // kernel pref + ::testing::Values(true, false) // memory test + ), + ::sgemmGenericSUPPrint() + ); +#endif + +#endif // defined(BLIS_KERNELS_ZEN5) && defined(GTEST_AVX512) + /*******************************************************/ /* Native Kernel testing */ /*******************************************************/ diff --git a/kernels/zen5/3/bli_gemm_tiny_zen5.c b/kernels/zen5/3/bli_gemm_tiny_zen5.c new file mode 100644 index 000000000..338c77052 --- /dev/null +++ b/kernels/zen5/3/bli_gemm_tiny_zen5.c @@ -0,0 +1,101 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +// Defining separate static arrays to hold all the kernel info, based on the datatype +static gemmtiny_ukr_info_t cgemmtiny_ukr_zen5[] = +{ + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, FALSE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, TRUE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, FALSE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, FALSE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, FALSE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, TRUE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, FALSE, 24, 4 }, + { (void *)bli_cgemmsup_cv_zen4_asm_24x4m, (void *)bli_cpackm_zen4_asm_24xk, FALSE, FALSE, 24, 4 } +}; + +static gemmtiny_ukr_info_t zgemmtiny_ukr_zen5[] = +{ + { (void *)bli_zgemmsup_cv_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cd_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cv_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cv_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cv_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cd_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cv_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 }, + { (void *)bli_zgemmsup_cv_zen4_asm_12x4m, (void *)bli_zpackm_zen4_asm_12xk, FALSE, FALSE, 12, 4 } +}; + +static gemmtiny_ukr_info_t sgemmtiny_ukr_zen5[] = +{ + { (void *)bli_sgemmsup_rv_zen4_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rd_zen5_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rv_zen4_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rv_zen4_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rv_zen4_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rd_zen5_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rv_zen4_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 }, + { (void *)bli_sgemmsup_rv_zen4_asm_6x64m, (void *)bli_spackm_haswell_asm_6xk, TRUE, FALSE, 6, 64 } +}; + +// Function macro that defines the bli_?gemmtiny_ukr_zen5_info functions +// These are used to acquire the kernel info at framework level +#undef GENTFUNC +#define GENTFUNC( ftype, ch, tfuncname ) \ +err_t PASTEMAC( ch, tfuncname ) \ + ( \ + stor3_t stor_id, \ + gemmtiny_ukr_info_t *fp_info \ + ) \ +{ \ + /* Acquire the object based on stor_id */ \ + *fp_info = TINY_GEMM_ZEN5(ch)[stor_id]; \ + /* If the kernel doesn't exist, return the appropriate signal */ \ + if ( fp_info->ukr_fp == NULL ) \ + { \ + return BLIS_NOT_YET_IMPLEMENTED; \ + } \ + else if ( ( fp_info->pack_fp == NULL ) && ( fp_info->enable_pack == TRUE ) ) \ + { \ + return BLIS_NOT_YET_IMPLEMENTED; \ + } \ +\ + return BLIS_SUCCESS; \ +} \ + +GENTFUNC( scomplex, c, gemmtiny_ukr_zen5_info ) +GENTFUNC( dcomplex, z, gemmtiny_ukr_zen5_info ) +GENTFUNC( float, s, gemmtiny_ukr_zen5_info ) \ No newline at end of file diff --git a/kernels/zen5/3/bli_gemm_tiny_zen5.h b/kernels/zen5/3/bli_gemm_tiny_zen5.h new file mode 100644 index 000000000..d12055c38 --- /dev/null +++ b/kernels/zen5/3/bli_gemm_tiny_zen5.h @@ -0,0 +1,59 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +// Macro to access the appropriate static array(that contains the kernel list), +// based on the datatype +#define TINY_GEMM_ZEN5(ch) ch ## gemmtiny_ukr_zen5 + +// Function macro signatures for bli_?gemmtiny_ukr_zen5_info functions +// These are used to acquire the kernel info at framework level +#undef GENTFUNC +#define GENTFUNC( ftype, ch, tfuncname ) \ +err_t PASTEMAC( ch, tfuncname ) \ + ( \ + stor3_t stor_id, \ + gemmtiny_ukr_info_t *fp_info \ + ); \ + +GENTFUNC( scomplex, c, gemmtiny_ukr_zen5_info ) +GENTFUNC( dcomplex, z, gemmtiny_ukr_zen5_info ) +GENTFUNC( float, s, gemmtiny_ukr_zen5_info ) + +/* Enabling the query for AVX512 kernels, based on the library's configuration */ +/* Minimum requirement is 'ZEN5' */ +#define LOOKUP_ZEN5_UKR( ch, stor_id, ukr_support, gemmtiny_ukr_info ) \ +{ \ + /* Call the appropriate function to query the AVX512 object info */ \ + ukr_support = PASTEMAC(ch, gemmtiny_ukr_zen5_info)( stor_id, &gemmtiny_ukr_info ); \ +} diff --git a/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.c b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.c new file mode 100644 index 000000000..ca717795c --- /dev/null +++ b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.c @@ -0,0 +1,5202 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +#define BLIS_ASM_SYNTAX_ATT +#include "bli_x86_asm_macros.h" + +#include "bli_gemmsup_rd_zen5_asm_s6x64.h" + +void bli_sgemmsup_rd_zen5_asm_5x64 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This file's x64 kernels only handle columns of C in groups of 4. + // Any remaining 1-3 columns of C are handled elsewhere. + // This kernel handles 5 rows of C over those 4-column groups. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns of C at a time; any remaining 1-3 columns of C + // are intentionally not handled here. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + uint64_t n_main_loop = n0 - n_left; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // This kernel only computes columns of C in the range [0, 4 * n_iter ). + // If n0 < 4, this kernel does no work and those columns of C are handled elsewhere. + if ( n_iter == 0 ) return; + + // Main 5x4 microkernel over the full left rectangle handled by this kernel: + // Handles 5 rows and columns in multiples of 4. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm28 ) // xmm28 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( n_main_loop ), r15 ) // r15 = n_main_loop + sub ( imm ( 4 ), r15 ) // jj = n_main_loop - 4 + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + lea ( mem ( r12, r15, 4 ), r12 ) // step to column jj within C + + lea ( mem ( , r15, 1 ), rsi ) // rsi = r15 = 4*jj; + imul ( r9, rsi ) // rsi *= cs_b; + lea ( mem ( rdx, rsi, 1 ), rdx ) // step to column jj within B + + mov ( r12, rcx ) // rcx = base of the current 5x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, r11, 4 ) ) // C row 4 + lea ( mem ( rcx, r11, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, r11, 1 ) ) // C row 3 + mov ( r14, rax ) // restart A at the top of this 4-col block + mov ( rdx, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm30 accumulate a 5x4 tile. + INIT_ACCUM_5x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 5 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 5x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 5 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 5x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + // Load one 16-float vector from each of the 5 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 5 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 5x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA5 ( 25, 17, 18, 19, 29, 30 ) + + label ( .POST_ACCUM ) + + // alpha is preloaded into xmm28 above the J-loop + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 5x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 5x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + // C_STOR then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 28, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR ( r11, 31, 4, 5, 6 ) // update the first 3 rows of the 5x4 tile + + // xmm22/xmm25 <- rows 3..4, cols 0..3. + ZMM_REDUCE_4 ( 20, 23, 26, 29, 22 ) // xmm22 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + ZMM_REDUCE_4 ( 21, 24, 27, 30, 25 ) // xmm25 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), sum ( zmm30 )] + + ALPHA_SCALE2 ( 28, 22, 25 ) // scale the next 2 rows by alpha + + C_STOR2_CONT ( r11, 31, 22, 25 ) // update the next 2 rows + + jmp ( .SDONE ) + + // Reduce the 5x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 28, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 22 ) // xmm22 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + ZMM_REDUCE_4 ( 21, 24, 27, 30, 25 ) // xmm25 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), sum ( zmm30 )] + + ALPHA_SCALE2 ( 28, 22, 25 ) // scale the next 2 rows by alpha + + C_STOR_BZ2_CONT ( r11, 22, 25 ) // store the next 2 rows + + label ( .SDONE ) + + sub ( imm ( 4 ), r15 ) + test ( r15, r15 ) + jns ( .SLOOP3X4J ) // iterate while jj >= 0 + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_main_loop] "m" ( n_main_loop ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm23", "ymm24", "ymm26", "ymm27", + "ymm29", "ymm30", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_4x64 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This file's x64 kernels only handle columns of C in groups of 4. + // Any remaining 1-3 columns of C are handled elsewhere. + // This kernel handles 4 rows of C over those 4-column groups. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns of C at a time; any remaining 1-3 columns of C + // are intentionally not handled here. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + uint64_t n_main_loop = n0 - n_left; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // This kernel only computes columns of C in the range [0, 4 * n_iter ). + // If n0 < 4, this kernel does no work and those columns of C are handled elsewhere. + if ( n_iter == 0 ) return; + + // Main 4x4 microkernel over the full rectangle handled by this kernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( n_main_loop ), r15 ) // r15 = n_main_loop + sub ( imm ( 4 ), r15 ) // jj = n_main_loop - 4 + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + lea ( mem ( r12, r15, 4 ), r12 ) // step to column jj within C + + lea ( mem ( , r15, 1 ), rsi ) // rsi = r15 = 4*jj; + imul ( r9, rsi ) // rsi *= cs_b; + lea ( mem ( rdx, rsi, 1 ), rdx ) // step to column jj within B + + mov ( r12, rcx ) // rcx = base of the current 4x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + lea ( mem ( rcx, r11, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, r11, 1 ) ) // C row 3 + mov ( r14, rax ) // restart A at the top of this 4-col block + mov ( rdx, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm29 accumulate a 4x4 tile. + INIT_ACCUM_4x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 4 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 4x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA4 ( 6, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA4 ( 7, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 4 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 4x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + // Load one 16-float vector from each of the 4 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 4 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 4x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA4 ( 25, 17, 18, 19, 29 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 4x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR ( r11, 31, 4, 5, 6 ) // update the first 3 rows of the 4x4 tile + + // xmm21 <- row 3, cols 0..3. + ZMM_REDUCE_4 ( 20, 23, 26, 29, 21 ) // xmm21 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + + ALPHA_SCALE1 ( 30, 21 ) // scale the next row by alpha + + C_STOR1_CONT ( r11, 31, 21 ) // update the next row + + jmp ( .SDONE ) + + // Reduce the 4x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 21 ) // xmm21 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + + ALPHA_SCALE1 ( 30, 21 ) // scale the next row by alpha + + C_STOR_BZ1_CONT ( r11, 21 ) // store the next row + + label ( .SDONE ) + + sub ( imm ( 4 ), r15 ) + test ( r15, r15 ) + jns ( .SLOOP3X4J ) // iterate while jj >= 0 + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_main_loop] "m" ( n_main_loop ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm23", "ymm26", "ymm29", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_3x64 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This file's x64 kernels only handle columns of C in groups of 4. + // Any remaining 1-3 columns of C are handled elsewhere. + // This kernel handles 3 rows of C over those 4-column groups. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns of C at a time; any remaining 1-3 columns of C + // are intentionally not handled here. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + uint64_t n_main_loop = n0 - n_left; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // This kernel only computes columns of C in the range [0, 4 * n_iter ). + // If n0 < 4, this kernel does no work and those columns of C are handled elsewhere. + if ( n_iter == 0 ) return; + + // Main 3x4 microkernel over the full rectangle handled by this kernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( n_main_loop ), r15 ) // r15 = n_main_loop + sub ( imm ( 4 ), r15 ) // jj = n_main_loop - 4 + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + lea ( mem ( r12, r15, 4 ), r12 ) // step to column jj within C + + lea ( mem ( , r15, 1 ), rsi ) // rsi = r15 = 4*jj; + imul ( r9, rsi ) // rsi *= cs_b; + lea ( mem ( rdx, rsi, 1 ), rdx ) // step to column jj within B + + mov ( r12, rcx ) // rcx = base of the current 3x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + mov ( r14, rax ) // restart A at the top of this 4-col block + mov ( rdx, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm19 accumulate a 3x4 tile. + INIT_ACCUM_3x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA3 ( 25, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA3 ( 25, 17, 18, 19 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA3 ( 22, 14, 15, 16 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA3 ( 25, 17, 18, 19 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 3x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR ( r11, 31, 4, 5, 6 ) // update the 3 rows of the 3x4 tile + + jmp ( .SDONE ) + + // Reduce the 3x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the 3 rows without reading C + + label ( .SDONE ) + + sub ( imm ( 4 ), r15 ) + test ( r15, r15 ) + jns ( .SLOOP3X4J ) // iterate while jj >= 0 + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_main_loop] "m" ( n_main_loop ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_2x64 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This file's x64 kernels only handle columns of C in groups of 4. + // Any remaining 1-3 columns of C are handled elsewhere. + // This kernel handles 2 rows of C over those 4-column groups. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns of C at a time; any remaining 1-3 columns of C + // are intentionally not handled here. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + uint64_t n_main_loop = n0 - n_left; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // This kernel only computes columns of C in the range [0, 4 * n_iter ). + // If n0 < 4, this kernel does no work and those columns of C are handled elsewhere. + if ( n_iter == 0 ) return; + + // Main 2x4 microkernel over the full rectangle handled by this kernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( n_main_loop ), r15 ) // r15 = n_main_loop + sub ( imm ( 4 ), r15 ) // jj = n_main_loop - 4 + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + lea ( mem ( r12, r15, 4 ), r12 ) // step to column jj within C + + lea ( mem ( , r15, 1 ), rsi ) // rsi = r15 = 4*jj; + imul ( r9, rsi ) // rsi *= cs_b; + lea ( mem ( rdx, rsi, 1 ), rdx ) // step to column jj within B + + mov ( r12, rcx ) // rcx = base of the current 2x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + mov ( r14, rax ) // restart A at the top of this 4-col block + mov ( rdx, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm18 accumulate a 2x4 tile. + INIT_ACCUM_2x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 2x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 2x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 2x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm25 ) + VFMA2 ( 25, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA2 ( 25, 17, 18 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA2 ( 22, 14, 15 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA2 ( 25, 17, 18 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 2x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // xmm4/xmm5 <- rows 0..1, cols 0..3. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR2 ( r11, 31, 4, 5 ) // update the 2 rows of the 2x4 tile + + jmp ( .SDONE ) + + // Reduce the 2x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR_BZ2 ( r11, 4, 5 ) // store the 2 rows without reading C + + label ( .SDONE ) + + sub ( imm ( 4 ), r15 ) + test ( r15, r15 ) + jns ( .SLOOP3X4J ) // iterate while jj >= 0 + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_main_loop] "m" ( n_main_loop ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm17", "ymm18", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_1x64 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This file's x64 kernels only handle columns of C in groups of 4. + // Any remaining 1-3 columns of C are handled elsewhere. + // This kernel handles 1 row of C over those 4-column groups. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns of C at a time; any remaining 1-3 columns of C + // are intentionally not handled here. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + uint64_t n_main_loop = n0 - n_left; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // This kernel only computes columns of C in the range [0, 4 * n_iter ). + // If n0 < 4, this kernel does no work and those columns of C are handled elsewhere. + if ( n_iter == 0 ) return; + + // Main 1x4 microkernel over the full rectangle handled by this kernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( n_main_loop ), r15 ) // r15 = n_main_loop + sub ( imm ( 4 ), r15 ) // jj = n_main_loop - 4 + mov ( var ( abuf ), r14 ) // load base address of a + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + lea ( mem ( r12, r15, 4 ), r12 ) // step to column jj within C + + lea ( mem ( , r15, 1 ), rsi ) // rsi = r15 = 4*jj; + imul ( r9, rsi ) // rsi *= cs_b; + lea ( mem ( rdx, rsi, 1 ), rdx ) // step to column jj within B + + mov ( r12, rcx ) // rcx = base of the current 1x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + mov ( r14, rax ) // restart A at the top of this 4-col block + mov ( rdx, rbx ) // restart B at the top of this 4-col block + + // zmm8/zmm11/zmm14/zmm17 accumulate a 1x4 tile. + INIT_ACCUM_1x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from the A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from the row of A. + vmovups ( ( rax ), zmm0 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 1x4 accumulators. + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm0, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm0, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm13 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vfmadd231ps ( ( rbx ), zmm13, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm13, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm13, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm13, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm15 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vfmadd231ps ( ( rbx ), zmm15, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm15, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm15, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm15, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm18 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vfmadd231ps ( ( rbx ), zmm18, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm18, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm18, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm18, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from the row of A. + vmovups ( ( rax ), zmm0 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 1x4 accumulators. + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm0, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm0, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm13 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vfmadd231ps ( ( rbx ), zmm13, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm13, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm13, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm13, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from the row of A. + vmovups ( ( rax ), zmm15 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 1x4 accumulators. + vfmadd231ps ( ( rbx ), zmm15, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm15, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm15, zmm14 ) + vfmadd231ps ( ( rbx, r13, 1 ), zmm15, zmm17 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA1 ( 6, 8 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA1 ( 7, 11 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA1 ( 22, 14 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA1 ( 25, 17 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA1 ( 6, 8 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA1 ( 7, 11 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA1 ( 22, 14 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 25 MASK_KZ ( 1 ) ) ) + VFMA1 ( 25, 17 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 1x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // xmm4 <- row 0, cols 0..3. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + + ALPHA_SCALE1 ( 30, 4 ) // scale the row by alpha + + C_STOR1 ( r11, 31, 4 ) // update the row of the 1x4 tile + + jmp ( .SDONE ) + + // Reduce the 1x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + + ALPHA_SCALE1 ( 30, 4 ) // scale the row by alpha + + C_STOR_BZ1 ( r11, 4 ) // store the row without reading C + + label ( .SDONE ) + + sub ( imm ( 4 ), r15 ) + test ( r15, r15 ) + jns ( .SLOOP3X4J ) // iterate while jj >= 0 + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_main_loop] "m" ( n_main_loop ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm6", + "ymm7", "ymm8", "ymm10", "ymm11", "ymm13", + "ymm14", "ymm17", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_5x3 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 5 rows and 3 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 5x3 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + mov ( imm ( 7 ), esi ) + kmovw ( esi, K ( 2 ) ) // k2 = mask for the 3 active C columns + + mov ( var ( abuf ), rax ) // load base address of a + mov ( var ( bbuf ), rbx ) // load base address of b + mov ( var ( cbuf ), rcx ) // load base address of c + + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, r11, 4 ) ) // C row 4 + lea ( mem ( rcx, r11, 2 ), rsi ) // r11 = rcx + 2 * rs_c + prefetchw0 ( mem ( rsi, r11, 1 ) ) // C row 3 + + INIT_ACCUM_5x3 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA5 ( 22, 14, 15, 16, 26, 27 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 5x3 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + ZMM_REDUCE_3 ( 10, 13, 16, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_MASKED ( r11, 31, 4, 5, 6 ) // update the first 3 rows of the 5x3 tile + + ZMM_REDUCE_3 ( 20, 23, 26, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 )] + ZMM_REDUCE_3 ( 21, 24, 27, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_MASKED2_CONT ( r11, 31, 17, 18 ) // update the next 2 rows + + jmp ( .SDONE ) + + // Reduce the 5x3 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + ZMM_REDUCE_3 ( 10, 13, 16, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_BZ_MASKED ( r11, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_3 ( 20, 23, 26, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 )] + ZMM_REDUCE_3 ( 21, 24, 27, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_BZ_MASKED2_CONT ( r11, 17, 18 ) // store the next 2 rows + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1", "k2" + ) +} + +void bli_sgemmsup_rd_zen5_asm_4x3 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 4 rows and 3 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 4x3 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + mov ( imm ( 7 ), esi ) + kmovw ( esi, K ( 2 ) ) // k2 = mask for the 3 active C columns + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + lea ( mem ( rcx, r11, 2 ), rsi ) // r11 = rcx + 2 * rs_c + prefetchw0 ( mem ( rsi, r11, 1 ) ) // C row 3 + + INIT_ACCUM_4x3 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA4 ( 22, 14, 15, 16, 26 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 4x3 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the first 2 rows by alpha + + C_STOR_MASKED2 ( r11, 31, 4, 5 ) // update the first 2 rows of the 4x3 tile + + ZMM_REDUCE_3 ( 10, 13, 16, 17 ) // xmm17 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 )] + ZMM_REDUCE_3 ( 20, 23, 26, 18 ) // xmm18 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_MASKED2_CONT ( r11, 31, 17, 18 ) // update the next 2 rows + + jmp ( .SDONE ) + + // Reduce the 4x3 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the first 2 rows by alpha + + C_STOR_BZ_MASKED2 ( r11, 4, 5 ) // store the first 2 rows without reading C + + ZMM_REDUCE_3 ( 10, 13, 16, 17 ) // xmm17 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 )] + ZMM_REDUCE_3 ( 20, 23, 26, 18 ) // xmm18 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_BZ_MASKED2_CONT ( r11, 17, 18 ) // store the next 2 rows + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1", "k2" + ) +} + +void bli_sgemmsup_rd_zen5_asm_3x3 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 3 rows and 3 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 3x3 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + mov ( imm ( 7 ), esi ) + kmovw ( esi, K ( 2 ) ) // k2 = mask for the 3 active C columns + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + + INIT_ACCUM_3x3 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA3 ( 22, 14, 15, 16 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA3 ( 22, 14, 15, 16 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA3 ( 22, 14, 15, 16 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 3x3 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + ZMM_REDUCE_3 ( 10, 13, 16, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_MASKED ( r11, 31, 4, 5, 6 ) // update the 3 rows of the 3x3 tile + + jmp ( .SDONE ) + + // Reduce the 3x3 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + ZMM_REDUCE_3 ( 10, 13, 16, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_BZ_MASKED ( r11, 4, 5, 6 ) // store the 3 rows without reading C + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1", "k2" + ) +} + +void bli_sgemmsup_rd_zen5_asm_2x3 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 2 rows and 3 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 2x3 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + mov ( imm ( 7 ), esi ) // k2 = keep the first 3 lanes for 3 columns of C + kmovw ( esi, K ( 2 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + + INIT_ACCUM_2x3 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm22 ) + VFMA2 ( 22, 14, 15 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA2 ( 22, 14, 15 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA2 ( 22, 14, 15 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 2x3 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR_MASKED2 ( r11, 31, 4, 5 ) // update the 2x3 tile in C + + jmp ( .SDONE ) + + // Reduce the 2x3 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR_BZ_MASKED2 ( r11, 4, 5 ) // store the 2x3 tile without reading C + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1", "k2" + ) +} + +void bli_sgemmsup_rd_zen5_asm_1x3 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 1 row and 3 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 1x3 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + mov ( imm ( 7 ), esi ) // k2 = keep the first 3 lanes for 3 columns of C + kmovw ( esi, K ( 2 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + prefetchw0 ( mem ( rcx ) ) // C row 0 + + INIT_ACCUM_1x3 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm0, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm13 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm13, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm13, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm13, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm15 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm15, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm15, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm15, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm16 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm16, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm16, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm16, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm0, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm13 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm13, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm13, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm13, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + + add ( imm ( 16*4 ), rax ) + + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + vfmadd231ps ( ( rbx, r9, 2 ), zmm0, zmm14 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA1 ( 6, 8 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA1 ( 7, 11 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA1 ( 22, 14 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA1 ( 6, 8 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA1 ( 7, 11 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 22 MASK_KZ ( 1 ) ) ) + VFMA1 ( 22, 14 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 1x3 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + + ALPHA_SCALE1 ( 30, 4 ) // scale the row by alpha + + C_STOR_MASKED1 ( r11, 31, 4 ) // update the 1x3 tile in C + + jmp ( .SDONE ) + + // Reduce the 1x3 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 )] + + ALPHA_SCALE1 ( 30, 4 ) // scale the row by alpha + + C_STOR_BZ_MASKED1 ( r11, 4 ) // store the 1x3 tile without reading C + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1", "k2" + ) +} + +void bli_sgemmsup_rd_zen5_asm_5x2 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 5 rows and 2 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 5x2 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, r11, 4 ) ) // C row 4 + lea ( mem ( rcx, r11, 2 ), rsi ) // r11 = rcx + 2 * rs_c + prefetchw0 ( mem ( rsi, r11, 1 ) ) // C row 3 + + INIT_ACCUM_5x2 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 21 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 23, 24 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 5x2 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + ZMM_REDUCE_2 ( 10, 13, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_2_FLOATS ( r11, 31, 4, 5, 6 ) // update the first 3 rows of the 5x2 tile + + ZMM_REDUCE_2 ( 20, 23, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 )] + ZMM_REDUCE_2 ( 21, 24, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_2_FLOATS2_CONT ( r11, 31, 17, 18 ) // update the next 2 rows + + jmp ( .SDONE ) + + // Reduce the 5x2 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + ZMM_REDUCE_2 ( 10, 13, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_BZ_2_FLOATS ( r11, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_2 ( 20, 23, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 )] + ZMM_REDUCE_2 ( 21, 24, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_BZ_2_FLOATS2_CONT ( r11, 17, 18 ) // store the next 2 rows + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_4x2 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 4 rows and 2 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 4x2 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + lea ( mem ( rcx, r11, 2 ), rsi ) // r11 = rcx + 2 * rs_c + prefetchw0 ( mem ( rsi, r11, 1 ) ) // C row 3 + + INIT_ACCUM_4x2 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 3x2 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the first 2 rows by alpha + + C_STOR_2_FLOATS2 ( r11, 31, 4, 5 ) // update the first 2 rows of the 4x2 tile + + ZMM_REDUCE_2 ( 10, 13, 17 ) // xmm17 = [sum ( zmm10 ), sum ( zmm13 )] + ZMM_REDUCE_2 ( 20, 23, 18 ) // xmm18 = [sum ( zmm20 ), sum ( zmm23 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_2_FLOATS2_CONT ( r11, 31, 17, 18 ) // update the next 2 rows + + jmp ( .SDONE ) + + // Reduce the 4x2 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the first 2 rows by alpha + + C_STOR_BZ_2_FLOATS2 ( r11, 4, 5 ) // store the first 2 rows without reading C + + ZMM_REDUCE_2 ( 10, 13, 17 ) // xmm17 = [sum ( zmm10 ), sum ( zmm13 )] + ZMM_REDUCE_2 ( 20, 23, 18 ) // xmm18 = [sum ( zmm20 ), sum ( zmm23 )] + + ALPHA_SCALE2 ( 30, 17, 18 ) // scale the next 2 rows by alpha + + C_STOR_BZ_2_FLOATS2_CONT ( r11, 17, 18 ) // store the next 2 rows + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_3x2 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 3 rows and 2 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 3x2 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + + INIT_ACCUM_3x2 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 2x2 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + ZMM_REDUCE_2 ( 10, 13, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_2_FLOATS ( r11, 31, 4, 5, 6 ) // update the 3x2 tile in C + + jmp ( .SDONE ) + + // Reduce the 3x2 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + ZMM_REDUCE_2 ( 10, 13, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_BZ_2_FLOATS ( r11, 4, 5, 6 ) // store the 3x2 tile without reading C + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_2x2 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 2 rows and 2 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 2x2 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + + INIT_ACCUM_2x2 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 2x2 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR_2_FLOATS2 ( r11, 31, 4, 5 ) // update the 2x2 tile in C + + jmp ( .SDONE ) + + // Reduce the 2x2 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR_BZ_2_FLOATS2 ( r11, 4, 5 ) // store the 2x2 tile without reading C + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_1x2 + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 1 row and 2 columns of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // Main 1x2 microkernel. + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rax ) // load address of a + mov ( var ( bbuf ), rbx ) // load address of b + mov ( var ( cbuf ), rcx ) // load address of c + + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + + prefetchw0 ( mem ( rcx ) ) // C row 0 + + INIT_ACCUM_1x2 + + mov ( var ( k_iter64 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm12 ) + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm12, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm12, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm13 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm13, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm13, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm14 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm14, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm14, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // load k_iter + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // ITER 0 + // load row from A + vmovups ( ( rax ), zmm0 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm12 ) + + add ( imm ( 16*4 ), rax ) + + // load column from B + vfmadd231ps ( ( rbx ), zmm12, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm12, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + + add ( imm ( 16*4 ), rax ) + + vfmadd231ps ( ( rbx ), zmm0, zmm8 ) + vfmadd231ps ( ( rbx, r9, 1 ), zmm0, zmm11 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA1 ( 6, 8 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA1 ( 7, 11 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA1 ( 6, 8 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA1 ( 7, 11 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 1x2 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + + ALPHA_SCALE1 ( 30, 4 ) // scale the row by alpha + + C_STOR_2_FLOATS1 ( r11, 31, 4 ) // update the 1x2 tile in C + + jmp ( .SDONE ) + + // Reduce the 1x2 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 )] + + ALPHA_SCALE1 ( 30, 4 ) // scale the row by alpha + + C_STOR_BZ_2_FLOATS1 ( r11, 4 ) // store the 1x2 tile without reading C + + label ( .SDONE ) + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} diff --git a/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.h b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.h new file mode 100644 index 000000000..9e08b00a6 --- /dev/null +++ b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64.h @@ -0,0 +1,700 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#define BLIS_ASM_SYNTAX_ATT +#include "bli_x86_asm_macros.h" + +/* Zero only accumulators for 4 B-columns (6x4 tile): zmm8-zmm31 */ +#define INIT_ACCUM_4COL \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm17,zmm17,zmm17 ) \ + vxorps( zmm18,zmm18,zmm18 ) \ + vxorps( zmm19,zmm19,zmm19 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm21,zmm21,zmm21 ) \ + vxorps( zmm22,zmm22,zmm22 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm24,zmm24,zmm24 ) \ + vxorps( zmm25,zmm25,zmm25 ) \ + vxorps( zmm26,zmm26,zmm26 ) \ + vxorps( zmm27,zmm27,zmm27 ) \ + vxorps( zmm28,zmm28,zmm28 ) \ + vxorps( zmm29,zmm29,zmm29 ) \ + vxorps( zmm30,zmm30,zmm30 ) \ + vxorps( zmm31,zmm31,zmm31 ) + +/* Zero only accumulators for 3 B-columns (6x3 tile): zmm8-16, zmm20-28 */ +#define INIT_ACCUM_3COL \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm21,zmm21,zmm21 ) \ + vxorps( zmm22,zmm22,zmm22 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm24,zmm24,zmm24 ) \ + vxorps( zmm25,zmm25,zmm25 ) \ + vxorps( zmm26,zmm26,zmm26 ) \ + vxorps( zmm27,zmm27,zmm27 ) \ + vxorps( zmm28,zmm28,zmm28 ) + +/* Zero only accumulators for 2 B-columns (6x2 tile): zmm8-13, zmm20-25 */ +#define INIT_ACCUM_2COL \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm21,zmm21,zmm21 ) \ + vxorps( zmm22,zmm22,zmm22 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm24,zmm24,zmm24 ) \ + vxorps( zmm25,zmm25,zmm25 ) + +/* 5-row x 4-B-col accumulators: zmm 8-10,11-13,14-16,17-19,20-21,23-24,26-27,29-30 (20 regs) */ +#define INIT_ACCUM_5x4 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm17,zmm17,zmm17 ) \ + vxorps( zmm18,zmm18,zmm18 ) \ + vxorps( zmm19,zmm19,zmm19 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm21,zmm21,zmm21 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm24,zmm24,zmm24 ) \ + vxorps( zmm26,zmm26,zmm26 ) \ + vxorps( zmm27,zmm27,zmm27 ) \ + vxorps( zmm29,zmm29,zmm29 ) \ + vxorps( zmm30,zmm30,zmm30 ) + +/* 4-row x 4-B-col accumulators: zmm 8-10,11-13,14-16,17-19,20,23,26,29 (16 regs) */ +#define INIT_ACCUM_4x4 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm17,zmm17,zmm17 ) \ + vxorps( zmm18,zmm18,zmm18 ) \ + vxorps( zmm19,zmm19,zmm19 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm26,zmm26,zmm26 ) \ + vxorps( zmm29,zmm29,zmm29 ) + +/* 3-row x 4-B-col accumulators: zmm 8-10,11-13,14-16,17-19 (12 regs) */ +#define INIT_ACCUM_3x4 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm17,zmm17,zmm17 ) \ + vxorps( zmm18,zmm18,zmm18 ) \ + vxorps( zmm19,zmm19,zmm19 ) + +/* 2-row x 4-B-col accumulators: zmm 8-9,11-12,14-15,17-18 (8 regs) */ +#define INIT_ACCUM_2x4 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm17,zmm17,zmm17 ) \ + vxorps( zmm18,zmm18,zmm18 ) + +/* 1-row x 4-B-col accumulators: zmm 8,11,14,17 (4 regs) */ +#define INIT_ACCUM_1x4 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm17,zmm17,zmm17 ) + +/* 5-row x 3-B-col accumulators: zmm 8-10,11-13,14-16,20-21,23-24,26-27 (15 regs) */ +#define INIT_ACCUM_5x3 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm21,zmm21,zmm21 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm24,zmm24,zmm24 ) \ + vxorps( zmm26,zmm26,zmm26 ) \ + vxorps( zmm27,zmm27,zmm27 ) + +/* 4-row x 3-B-col accumulators: zmm 8-10,11-13,14-16,20,23,26 (12 regs) */ +#define INIT_ACCUM_4x3 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm26,zmm26,zmm26 ) + +/* 3-row x 3-B-col accumulators: zmm 8-10,11-13,14-16 (9 regs) */ +#define INIT_ACCUM_3x3 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) \ + vxorps( zmm16,zmm16,zmm16 ) + +/* 2-row x 3-B-col accumulators: zmm 8-9,11-12,14-15 (6 regs) */ +#define INIT_ACCUM_2x3 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm14,zmm14,zmm14 ) \ + vxorps( zmm15,zmm15,zmm15 ) + +/* 1-row x 3-B-col accumulators: zmm 8,11,14 (3 regs) */ +#define INIT_ACCUM_1x3 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm14,zmm14,zmm14 ) + +/* 5-row x 2-B-col accumulators: zmm 8-10,11-13,20-21,23-24 (10 regs) */ +#define INIT_ACCUM_5x2 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm21,zmm21,zmm21 ) \ + vxorps( zmm23,zmm23,zmm23 ) \ + vxorps( zmm24,zmm24,zmm24 ) + +/* 4-row x 2-B-col accumulators: zmm 8-10,11-13,20,23 (8 regs) */ +#define INIT_ACCUM_4x2 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) \ + vxorps( zmm20,zmm20,zmm20 ) \ + vxorps( zmm23,zmm23,zmm23 ) + +/* 3-row x 2-B-col accumulators: zmm 8-10,11-13 (6 regs) */ +#define INIT_ACCUM_3x2 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm10,zmm10,zmm10 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) \ + vxorps( zmm13,zmm13,zmm13 ) + +/* 2-row x 2-B-col accumulators: zmm 8-9,11-12 (4 regs) */ +#define INIT_ACCUM_2x2 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm9,zmm9,zmm9 ) \ + vxorps( zmm11,zmm11,zmm11 ) \ + vxorps( zmm12,zmm12,zmm12 ) + +/* 1-row x 2-B-col accumulators: zmm 8,11 (2 regs) */ +#define INIT_ACCUM_1x2 \ + vxorps( zmm8,zmm8,zmm8 ) \ + vxorps( zmm11,zmm11,zmm11 ) + +#define VFMA6( Rload, R0, R1, R2, R3, R4, R5 ) \ + vfmadd231ps( zmm0, zmm(Rload), zmm(R0) ) /* zmm(R0) += zmm0 * zmm(Rload) */ \ + vfmadd231ps( zmm1, zmm(Rload), zmm(R1) ) /* zmm(R1) += zmm1 * zmm(Rload) */ \ + vfmadd231ps( zmm2, zmm(Rload), zmm(R2) ) /* zmm(R2) += zmm2 * zmm(Rload) */ \ + vfmadd231ps( zmm3, zmm(Rload), zmm(R3) ) /* zmm(R3) += zmm3 * zmm(Rload) */ \ + vfmadd231ps( zmm4, zmm(Rload), zmm(R4) ) /* zmm(R4) += zmm4 * zmm(Rload) */ \ + vfmadd231ps( zmm5, zmm(Rload), zmm(R5) ) /* zmm(R5) += zmm5 * zmm(Rload) */ + +#define VFMA5( Rload, R0, R1, R2, R3, R4 ) \ + vfmadd231ps( zmm0, zmm(Rload), zmm(R0) ) /* zmm(R0) += zmm0 * zmm(Rload) */ \ + vfmadd231ps( zmm1, zmm(Rload), zmm(R1) ) /* zmm(R1) += zmm1 * zmm(Rload) */ \ + vfmadd231ps( zmm2, zmm(Rload), zmm(R2) ) /* zmm(R2) += zmm2 * zmm(Rload) */ \ + vfmadd231ps( zmm3, zmm(Rload), zmm(R3) ) /* zmm(R3) += zmm3 * zmm(Rload) */ \ + vfmadd231ps( zmm4, zmm(Rload), zmm(R4) ) /* zmm(R4) += zmm4 * zmm(Rload) */ + +#define VFMA4( Rload, R0, R1, R2, R3 ) \ + vfmadd231ps( zmm0, zmm(Rload), zmm(R0) ) /* zmm(R0) += zmm0 * zmm(Rload) */ \ + vfmadd231ps( zmm1, zmm(Rload), zmm(R1) ) /* zmm(R1) += zmm1 * zmm(Rload) */ \ + vfmadd231ps( zmm2, zmm(Rload), zmm(R2) ) /* zmm(R2) += zmm2 * zmm(Rload) */ \ + vfmadd231ps( zmm3, zmm(Rload), zmm(R3) ) /* zmm(R3) += zmm3 * zmm(Rload) */ + +#define VFMA3( Rload, R0, R1, R2 ) \ + vfmadd231ps( zmm0, zmm(Rload), zmm(R0) ) /* zmm(R0) += zmm0 * zmm(Rload) */ \ + vfmadd231ps( zmm1, zmm(Rload), zmm(R1) ) /* zmm(R1) += zmm1 * zmm(Rload) */ \ + vfmadd231ps( zmm2, zmm(Rload), zmm(R2) ) /* zmm(R2) += zmm2 * zmm(Rload) */ + +#define VFMA2( Rload, R0, R1 ) \ + vfmadd231ps( zmm0, zmm(Rload), zmm(R0) ) /* zmm(R0) += zmm0 * zmm(Rload) */ \ + vfmadd231ps( zmm1, zmm(Rload), zmm(R1) ) /* zmm(R1) += zmm1 * zmm(Rload) */ + +#define VFMA1( Rload, R0 ) \ + vfmadd231ps( zmm0, zmm(Rload), zmm(R0) ) /* zmm(R0) += zmm0 * zmm(Rload) */ + +/* zmm(R0)=[a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15] */ +/* zmm(R1)=[b0,b1,b2,b3,b4,b5,b6,b7,b8,b9,b10,b11,b12,b13,b14,b15] */ +/* zmm(R2)=[c0,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15] */ +/* zmm(R3)=[d0,d1,d2,d3,d4,d5,d6,d7,d8,d9,d10,d11,d12,d13,d14,d15] */ +/* Input: four 16-float accumulators in zmm(R0) through zmm(R3). */ +/* Output: four final sums packed as [sum(a),sum(b),sum(c),sum(d)] in xmm(RD). */ +#define ZMM_REDUCE_4(R0, R1, R2, R3, RD) \ + vshufps( imm(0x88), zmm(R1), zmm(R0), zmm0 ) /* zmm0=[a0,a2,b0,b2,a4,a6,b4,b6, */ \ + /* a8,a10,b8,b10,a12,a14,b12,b14] */ \ + vshufps( imm(0xDD), zmm(R1), zmm(R0), zmm1 ) /* zmm1=[a1,a3,b1,b3,a5,a7,b5,b7, */ \ + /* a9,a11,b9,b11,a13,a15,b13,b15] */ \ + vaddps( zmm1, zmm0, zmm0 ) /* zmm0=[a0+a1,a2+a3,b0+b1,b2+b3, */ \ + /* a4+a5,a6+a7,b4+b5,b6+b7, */ \ + /* a8+a9,a10+a11,b8+b9,b10+b11, */ \ + /* a12+a13,a14+a15,b12+b13,b14+b15] */ \ + vshufps( imm(0x88), zmm(R3), zmm(R2), zmm2 ) /* zmm2=[c0,c2,d0,d2,c4,c6,d4,d6, */ \ + /* c8,c10,d8,d10,c12,c14,d12,d14] */ \ + vshufps( imm(0xDD), zmm(R3), zmm(R2), zmm3 ) /* zmm3=[c1,c3,d1,d3,c5,c7,d5,d7, */ \ + /* c9,c11,d9,d11,c13,c15,d13,d15] */ \ + vaddps( zmm3, zmm2, zmm2 ) /* zmm2=[c0+c1,c2+c3,d0+d1,d2+d3, */ \ + /* c4+c5,c6+c7,d4+d5,d6+d7, */ \ + /* c8+c9,c10+c11,d8+d9,d10+d11, */ \ + /* c12+c13,c14+c15,d12+d13,d14+d15] */ \ + vshufps( imm(0x88), zmm2, zmm0, zmm1 ) /* zmm1=[a0+a1,b0+b1,c0+c1,d0+d1, */ \ + /* a4+a5,b4+b5,c4+c5,d4+d5, */ \ + /* a8+a9,b8+b9,c8+c9,d8+d9, */ \ + /* a12+a13,b12+b13,c12+c13,d12+d13] */ \ + vshufps( imm(0xDD), zmm2, zmm0, zmm3 ) /* zmm3=[a2+a3,b2+b3,c2+c3,d2+d3, */ \ + /* a6+a7,b6+b7,c6+c7,d6+d7, */ \ + /* a10+a11,b10+b11,c10+c11,d10+d11, */ \ + /* a14+a15,b14+b15,c14+c15,d14+d15] */ \ + vaddps( zmm3, zmm1, zmm1 ) /* zmm1=[a0+...+a3,b0+...+b3,c0+...+c3,d0+...+d3, */ \ + /* a4+...+a7,b4+...+b7,c4+...+c7,d4+...+d7, */ \ + /* a8+...+a11,b8+...+b11,c8+...+c11,d8+...+d11, */ \ + /* a12+...+a15,b12+...+b15,c12+...+c15,d12+...+d15] */ \ + /* zmm1 now holds four quarter-sums for each output row. */ \ + vextractf32x8( imm(0x01), zmm1, ymm0 ) /* ymm0=[a8+...+a11,b8+...+b11,c8+...+c11,d8+...+d11, */ \ + /* a12+...+a15,b12+...+b15,c12+...+c15,d12+...+d15] */ \ + vaddps( ymm1, ymm0, ymm0 ) /* ymm0=[a0+...+a3+a8+...+a11,b0+...+b3+b8+...+b11, */ \ + /* c0+...+c3+c8+...+c11,d0+...+d3+d8+...+d11, */ \ + /* a4+...+a7+a12+...+a15,b4+...+b7+b12+...+b15, */ \ + /* c4+...+c7+c12+...+c15,d4+...+d7+d12+...+d15] */ \ + vextractf128( imm(0x01), ymm0, xmm1 ) /* xmm1=[a4+...+a7+a12+...+a15,b4+...+b7+b12+...+b15, */ \ + /* c4+...+c7+c12+...+c15,d4+...+d7+d12+...+d15] */ \ + vaddps( xmm0, xmm1, xmm(RD) ) /* xmm(RD)=[sum(a),sum(b),sum(c),sum(d)] */ + +/* Input: three 16-float accumulators in zmm(R0), zmm(R1), and zmm(R2). */ +/* Output: [sum(a),sum(b),sum(c),sum(c)] in xmm(RD). */ +#define ZMM_REDUCE_3(R0, R1, R2, RD) \ + vshufps( imm(0x88), zmm(R1), zmm(R0), zmm0 ) /* zmm0=[a0,a2,b0,b2,a4,a6,b4,b6, */ \ + /* a8,a10,b8,b10,a12,a14,b12,b14] */ \ + vshufps( imm(0xDD), zmm(R1), zmm(R0), zmm1 ) /* zmm1=[a1,a3,b1,b3,a5,a7,b5,b7, */ \ + /* a9,a11,b9,b11,a13,a15,b13,b15] */ \ + vaddps( zmm1, zmm0, zmm0 ) /* zmm0=[a0+a1,a2+a3,b0+b1,b2+b3, */ \ + /* a4+a5,a6+a7,b4+b5,b6+b7, */ \ + /* a8+a9,a10+a11,b8+b9,b10+b11, */ \ + /* a12+a13,a14+a15,b12+b13,b14+b15] */ \ + vshufps( imm(0x88), zmm(R2), zmm(R2), zmm2 ) /* zmm2=[c0,c2,c0,c2,c4,c6,c4,c6, */ \ + /* c8,c10,c8,c10,c12,c14,c12,c14] */ \ + vshufps( imm(0xDD), zmm(R2), zmm(R2), zmm3 ) /* zmm3=[c1,c3,c1,c3,c5,c7,c5,c7, */ \ + /* c9,c11,c9,c11,c13,c15,c13,c15] */ \ + vaddps( zmm3, zmm2, zmm2 ) /* zmm2=[c0+c1,c2+c3,c0+c1,c2+c3, */ \ + /* c4+c5,c6+c7,c4+c5,c6+c7, */ \ + /* c8+c9,c10+c11,c8+c9,c10+c11, */ \ + /* c12+c13,c14+c15,c12+c13,c14+c15] */ \ + vshufps( imm(0x88), zmm2, zmm0, zmm1 ) /* zmm1=[a0+a1,b0+b1,c0+c1,c0+c1, */ \ + /* a4+a5,b4+b5,c4+c5,c4+c5, */ \ + /* a8+a9,b8+b9,c8+c9,c8+c9, */ \ + /* a12+a13,b12+b13,c12+c13,c12+c13] */ \ + vshufps( imm(0xDD), zmm2, zmm0, zmm3 ) /* zmm3=[a2+a3,b2+b3,c2+c3,c2+c3, */ \ + /* a6+a7,b6+b7,c6+c7,c6+c7, */ \ + /* a10+a11,b10+b11,c10+c11,c10+c11, */ \ + /* a14+a15,b14+b15,c14+c15,c14+c15] */ \ + vaddps( zmm3, zmm1, zmm1 ) /* zmm1=[a0+...+a3,b0+...+b3,c0+...+c3,c0+...+c3, */ \ + /* a4+...+a7,b4+...+b7,c4+...+c7,c4+...+c7, */ \ + /* a8+...+a11,b8+...+b11,c8+...+c11,c8+...+c11, */ \ + /* a12+...+a15,b12+...+b15,c12+...+c15,c12+...+c15] */ \ + vextractf32x8( imm(0x01), zmm1, ymm0 ) /* ymm0=[a8+...+a11,b8+...+b11,c8+...+c11,c8+...+c11, */ \ + /* a12+...+a15,b12+...+b15,c12+...+c15,c12+...+c15] */ \ + vaddps( ymm1, ymm0, ymm0 ) /* ymm0=[a0+...+a3+a8+...+a11,b0+...+b3+b8+...+b11, */ \ + /* c0+...+c3+c8+...+c11,c0+...+c3+c8+...+c11, */ \ + /* a4+...+a7+a12+...+a15,b4+...+b7+b12+...+b15, */ \ + /* c4+...+c7+c12+...+c15,c4+...+c7+c12+...+c15] */ \ + vextractf128( imm(0x01), ymm0, xmm1 ) /* xmm1=[a4+...+a7+a12+...+a15,b4+...+b7+b12+...+b15, */ \ + /* c4+...+c7+c12+...+c15,c4+...+c7+c12+...+c15] */ \ + vaddps( xmm0, xmm1, xmm(RD) ) /* xmm(RD)=[sum(a),sum(b),sum(c),sum(c)] */ + +/* Input: two 16-float accumulators in zmm(R0) and zmm(R1). */ +/* Output: [sum(a),sum(b),sum(a),sum(b)] in xmm(RD). */ +#define ZMM_REDUCE_2(R0, R1, RD) \ + vshufps( imm(0x88), zmm(R1), zmm(R0), zmm0 ) /* zmm0=[a0,a2,b0,b2,a4,a6,b4,b6, */ \ + /* a8,a10,b8,b10,a12,a14,b12,b14] */ \ + vshufps( imm(0xDD), zmm(R1), zmm(R0), zmm1 ) /* zmm1=[a1,a3,b1,b3,a5,a7,b5,b7, */ \ + /* a9,a11,b9,b11,a13,a15,b13,b15] */ \ + vaddps( zmm1, zmm0, zmm0 ) /* zmm0=[a0+a1,a2+a3,b0+b1,b2+b3, */ \ + /* a4+a5,a6+a7,b4+b5,b6+b7, */ \ + /* a8+a9,a10+a11,b8+b9,b10+b11, */ \ + /* a12+a13,a14+a15,b12+b13,b14+b15] */ \ + vextractf32x8( imm(0x01), zmm0, ymm1 ) /* ymm1=[a8+a9,a10+a11,b8+b9,b10+b11, */ \ + /* a12+a13,a14+a15,b12+b13,b14+b15] */ \ + vaddps( ymm0, ymm1, ymm0 ) /* ymm0=[a0+a1+a8+a9,a2+a3+a10+a11, */ \ + /* b0+b1+b8+b9,b2+b3+b10+b11, */ \ + /* a4+a5+a12+a13,a6+a7+a14+a15, */ \ + /* b4+b5+b12+b13,b6+b7+b14+b15] */ \ + vextractf128( imm(0x01), ymm0, xmm1 ) /* xmm1=[a4+a5+a12+a13,a6+a7+a14+a15, */ \ + /* b4+b5+b12+b13,b6+b7+b14+b15] */ \ + vaddps( xmm0, xmm1, xmm0 ) /* xmm0=[a0+a1+a4+a5+a8+a9+a12+a13, */ \ + /* a2+a3+a6+a7+a10+a11+a14+a15, */ \ + /* b0+b1+b4+b5+b8+b9+b12+b13, */ \ + /* b2+b3+b6+b7+b10+b11+b14+b15] */ \ + vshufps( imm(0x88), xmm0, xmm0, xmm(RD) ) /* xmm(RD)=[a0+a1+a4+a5+a8+a9+a12+a13, */ \ + /* b0+b1+b4+b5+b8+b9+b12+b13, */ \ + /* a0+a1+a4+a5+a8+a9+a12+a13, */ \ + /* b0+b1+b4+b5+b8+b9+b12+b13] */ \ + vshufps( imm(0xDD), xmm0, xmm0, xmm3 ) /* xmm3=[a2+a3+a6+a7+a10+a11+a14+a15, */ \ + /* b2+b3+b6+b7+b10+b11+b14+b15, */ \ + /* a2+a3+a6+a7+a10+a11+a14+a15, */ \ + /* b2+b3+b6+b7+b10+b11+b14+b15] */ \ + vaddps( xmm3, xmm(RD), xmm(RD) ) /* xmm(RD)=[sum(a),sum(b),sum(a),sum(b)] */ + +#define C_STOR_BZ_2_FLOATS(R_rs_c, R0, R1, R2) \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovlps( xmm(R2), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R2)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_2_FLOATS1(R_rs_c, R0) \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_2_FLOATS2(R_rs_c, R0, R1) \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_MASKED(R_rs_c, R0, R1, R2) \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R2), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_MASKED2(R_rs_c, R0, R1) \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_MASKED1(R_rs_c, R0) \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ(R_rs_c, R0, R1, R2) \ + vmovups( xmm(R0), mem( rcx ) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem( rcx ) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R2), mem( rcx ) ) /* mem(rcx) <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ2(R_rs_c, R0, R1) \ + vmovups( xmm(R0), mem( rcx ) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem( rcx ) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ1(R_rs_c, R0) \ + vmovups( xmm(R0), mem( rcx ) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_CONT(R_rs_c, R0, R1, R2) \ + vmovups( xmm(R0), mem( rcx ) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem( rcx ) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R2), mem( rcx ) ) /* mem(rcx) <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ2_CONT(R_rs_c, R0, R1) \ + vmovups( xmm(R0), mem( rcx ) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem( rcx ) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ1_CONT(R_rs_c, R0) \ + vmovups( xmm(R0), mem( rcx ) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_MASKED_CONT(R_rs_c, R0, R1, R2) \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R2), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_MASKED2_CONT(R_rs_c, R0, R1) \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_2_FLOATS_CONT(R_rs_c, R0, R1, R2) \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovlps( xmm(R2), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R2)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_BZ_2_FLOATS2_CONT(R_rs_c, R0, R1) \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define ALPHA_SCALE(Ralpha, R0, R1, R2) \ + vmulps( xmm(Ralpha), xmm(R0), xmm(R0) ) /* xmm(R0) = alpha * xmm(R0) */ \ + vmulps( xmm(Ralpha), xmm(R1), xmm(R1) ) /* xmm(R1) = alpha * xmm(R1) */ \ + vmulps( xmm(Ralpha), xmm(R2), xmm(R2) ) /* xmm(R2) = alpha * xmm(R2) */ + +#define C_STOR(R_rs_c, Rbeta, R0, R1, R2) \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), (rcx) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), (rcx) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R2) ) /* xmm(R2) = beta * C + xmm(R2) */ \ + vmovups( xmm(R2), (rcx) ) /* mem(rcx) <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_CONT(R_rs_c, Rbeta, R0, R1, R2) \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), (rcx) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), (rcx) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R2) ) /* xmm(R2) = beta * C + xmm(R2) */ \ + vmovups( xmm(R2), (rcx) ) /* mem(rcx) <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_MASKED(R_rs_c, Rbeta, R0, R1, R2) \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R2) ) /* xmm(R2) = beta * C + xmm(R2) */ \ + vmovups( xmm(R2), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_MASKED_CONT(R_rs_c, Rbeta, R0, R1, R2) \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R2) ) /* xmm(R2) = beta * C + xmm(R2) */ \ + vmovups( xmm(R2), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R2) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_2_FLOATS(R_rs_c, Rbeta, R0, R1, R2) \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R2) ) /* xmm(R2) = beta * C + xmm(R2) */ \ + vmovlps( xmm(R2), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R2)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_2_FLOATS_CONT(R_rs_c, Rbeta, R0, R1, R2) \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R2) ) /* xmm(R2) = beta * C + xmm(R2) */ \ + vmovlps( xmm(R2), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R2)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define ALPHA_SCALE2(Ralpha, R0, R1) \ + vmulps( xmm(Ralpha), xmm(R0), xmm(R0) ) /* xmm(R0) = alpha * xmm(R0) */ \ + vmulps( xmm(Ralpha), xmm(R1), xmm(R1) ) /* xmm(R1) = alpha * xmm(R1) */ + +#define ALPHA_SCALE1(Ralpha, R0) \ + vmulps( xmm(Ralpha), xmm(R0), xmm(R0) ) /* xmm(R0) = alpha * xmm(R0) */ + +#define C_STOR2_CONT(R_rs_c, Rbeta, R0, R1) \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), (rcx) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), (rcx) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR1_CONT(R_rs_c, Rbeta, R0) \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), (rcx) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_MASKED2(R_rs_c, Rbeta, R0, R1) \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_MASKED2_CONT(R_rs_c, Rbeta, R0, R1) \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_2_FLOATS2(R_rs_c, Rbeta, R0, R1) \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_2_FLOATS2_CONT(R_rs_c, Rbeta, R0, R1) \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vmovsd( mem(rcx), xmm1 ) /* xmm1 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovlps( xmm(R1), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R1)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR2(R_rs_c, Rbeta, R0, R1) \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), (rcx) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), (rcx) ) /* mem(rcx) <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR1(R_rs_c, Rbeta, R0) \ + vfmadd231ps( (rcx), xmm(Rbeta), xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovups( xmm(R0), (rcx) ) /* mem(rcx) <- xmm(R0) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_MASKED1(R_rs_c, Rbeta, R1) \ + vmovups( mem(rcx), xmm(1 MASK_KZ(2)) ) /* xmm1 <- mem(rcx){k2,z} */ \ + vfmadd231ps( xmm(Rbeta), xmm1, xmm(R1) ) /* xmm(R1) = beta * C + xmm(R1) */ \ + vmovups( xmm(R1), mem(rcx) MASK_K(2) ) /* mem(rcx){k2} <- xmm(R1) */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ + +#define C_STOR_2_FLOATS1(R_rs_c, Rbeta, R0) \ + vmovsd( mem(rcx), xmm0 ) /* xmm0 <- mem(rcx)[0:1] */ \ + vfmadd231ps( xmm(Rbeta), xmm0, xmm(R0) ) /* xmm(R0) = beta * C + xmm(R0) */ \ + vmovlps( xmm(R0), mem(rcx) ) /* mem(rcx)[0:1] <- xmm(R0)[0:1] */ \ + add( R_rs_c, rcx ) /* rcx += rs_c * sizeof(float) */ diff --git a/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64m.c b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64m.c new file mode 100644 index 000000000..ac8b46a84 --- /dev/null +++ b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64m.c @@ -0,0 +1,1712 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +#define BLIS_ASM_SYNTAX_ATT +#include "bli_x86_asm_macros.h" + +#include "bli_gemmsup_rd_zen5_asm_s6x64.h" + +void bli_sgemmsup_rd_zen5_asm_6x64m + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // Main loop handles 4 columns at a time; 1-3 columns fall through to n_left dispatch. + uint64_t n_left = n0 % 4; + + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + uint64_t b_step_base = 0; + + uint64_t n_iter = n0 / 4; + uint64_t n_main_loop = n0 - n_left; + uint64_t j_step_end = 0; + uint64_t j_step_start = n_main_loop >= 1 ? ( ( n_main_loop - 1 ) / 16 ) * 16 : 0; + + // Main loop handles 6 rows at a time; 1-5 rows fall through to m_left dispatch. + uint64_t m_iter = m0 / 6; + uint64_t m_left = m0 % 6; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + // The decomposition into edge kernels is handled as follows: + // Example, m0 = 7 and n0 = 5: + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ m_left ][ n_left ] + // The asm handles all multiples of 6 rows and 4 columns in the top-left of the matrix + // Next, any remaining m_left rows are handled by a separate edge microkernel after the + // assembly section which still processes 4 columns at a time in the n dimension. + // Finally, any remaining n_left columns are handled by a separate edge microkernel + // that processes the remaining right side columns. + // n_left dispatch: handle the right edge columns that remain after the asm + // kernel covers all full 4-column blocks. + // This path owns the entire right-edge strip across rows [0, m0 ), including + // the bottom-right corner that the x64 m_left kernels intentionally skip. + // Computes the n_left part of C. + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + if ( n_left ) + { + float* restrict cij = c; + float* restrict bj = b; + float* restrict ai = a; + + // Start at column 4 * n_iter, i.e. the first column not covered by asm. + cij += n_main_loop*cs_c0; + bj += n_main_loop*cs_b0; + + if ( 3 == n_left ) + { + const dim_t nr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_6x3m + ( + conja, conjb, m0, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == n_left ) + { + const dim_t nr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_6x2m + ( + conja, conjb, m0, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == n_left ) + { + bli_sgemv_ex + ( + BLIS_NO_TRANSPOSE, conjb, m0, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, + beta, cij, rs_c0, cntx, NULL + ); + } + } + + // The asm microkernel only handles full 6x4 tiles. + // If either dimension is already in edge territory, skip straight to the + // C dispatch below. + if ( m_iter == 0 || n_iter == 0 ) goto consider_edge_cases; + + // Main 6x4 microkernel over the full top-left rectangle: + // Handles all rows in multiples of 6, and columns in multiples of 4. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ][ asm ][ - ] + // [ asm ][ asm ][ asm ][ asm ][ - ] + // [ asm ][ asm ][ asm ][ asm ][ - ] + // [ asm ][ asm ][ asm ][ asm ][ - ] + // [ asm ][ asm ][ asm ][ asm ][ - ] + // [ asm ][ asm ][ asm ][ asm ][ - ] + // [ - ][ - ] + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + // Outer J-step loop: process 16 columns ( 4 groups of 4 ) at a time. + // For each 16-col step, sweep all m_iter row-blocks ( full SLOOP3X4I ), + // then advance to next 16-col step. This is done so that writes happen to the full + // cache line in the C matrix + label ( .SLOOP3X4J_STEP ) // Outer loop: 16-col steps + mov ( var ( j_step_start ), r15 ) // r15 = j_step_start + mov ( imm ( 16 ), rsi ) // rsi = 16 + add ( r15, rsi ) // rsi = j_step_start + 16 ( desired end ) + mov ( var ( n_main_loop ), rax ) // rax = n_main_loop ( max bound ) + cmp ( rax, rsi ) // compare rsi vs n_main_loop + cmovg ( rax, rsi ) // if rsi > n_main_loop, rsi = n_main_loop + mov ( rsi, var ( j_step_end ) ) // j_step_end = min ( j_step_start + 16, n_main_loop ) + + mov ( var ( m_iter ), r11 ) // ii = m_iter; + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( cbuf ), r12 ) // load base address of c + + // Compute B base for this 16-col step once, outside the I-loop. + mov ( var ( bbuf ), rdx ) + sub ( imm ( 4 ), rsi ) // rsi = j_step_end - 4 + imul ( r9, rsi ) // rsi = ( j_step_end - 4 ) * cs_b_bytes + add ( rsi, rdx ) // rdx = B base for rightmost 4-col group + mov ( rdx, var ( b_step_base ) ) // save B base for I-loop reuse + + label ( .SLOOP3X4I ) // Middle loop over 6-row blocks of C/A + + // B base for this 16-col step: bbuf + j_step_start * cs_b_bytes + mov ( var ( b_step_base ), rdx ) // reload B base for the beginning of the I-loop + mov ( var ( j_step_end ), r15 ) + sub ( imm ( 4 ), r15 ) // r15 = j_step_end - 4 + + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + // rcx = base of the current 6x4 output tile in C. + lea ( mem ( r12, r15, 4 ), rcx ) // rcx = c_row_base + jj * sizeof ( float ) + + // Prefetch the rows of C that will be written to in this loop + mov ( var ( rs_c ), rsi ) // rsi = rs_c + lea ( mem ( , rsi, 4 ), rsi ) // rsi = rs_c * sizeof ( float ) + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, rsi, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, rsi, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, rsi, 4 ) ) // C row 4 + lea ( mem ( rcx, rsi, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, rsi, 1 ) ) // C row 3 = rcx + 3 * rs_c + lea ( mem ( rcx, rsi, 4 ), rax ) // rax = rcx + 4 * rs_c + prefetchw0 ( mem ( rax, rsi, 1 ) ) // C row 5 = rcx + 5 * rs_c + + mov ( r14, rax ) // restart A at the top of this row-block + mov ( rdx, rbx ) // load b to rbx + lea ( mem ( r10, r8, 2 ), rdi ) // rdi = 5 * rs_a + + // zmm8-zmm31 accumulate a 6x4 tile. + INIT_ACCUM_4COL + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 6x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 6x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), ZMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), YMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + label ( .POST_ACCUM ) + + // Preload alpha into xmm7 to avoid redundant memory reloads. + // xmm7 survives ZMM_REDUCE calls ( they only clobber zmm0-3 ). + mov ( var ( alpha ), rax ) // load address of alpha + vbroadcastss ( ( rax ), xmm7 ) // xmm7 = alpha + mov ( var ( beta ), rax ) // load address of beta + vbroadcastss ( ( rax ), xmm0 ) + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm0 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 6x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 6x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + // C_STOR then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 7, 4, 5, 6 ) // scale the first 3 rows by alpha + + // Preload beta into xmm9 now that zmm9 is free ( consumed by ZMM_REDUCE_4 ). + mov ( var ( beta ), rax ) + vbroadcastss ( ( rax ), xmm9 ) // xmm9 = beta + mov ( var ( rs_c ), rdi ) // rdi = rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rdi = rs_c * sizeof ( float ) + + C_STOR ( rdi, 9, 4, 5, 6 ) // update the first 3 rows of the 6x4 tile + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + ZMM_REDUCE_4 ( 21, 24, 27, 30, 11 ) // xmm11 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), sum ( zmm30 )] + ZMM_REDUCE_4 ( 22, 25, 28, 31, 14 ) // xmm14 = [sum ( zmm22 ), sum ( zmm25 ), sum ( zmm28 ), sum ( zmm31 )] + + ALPHA_SCALE ( 7, 8, 11, 14 ) // scale the next 3 rows by alpha + + C_STOR_CONT ( rdi, 9, 8, 11, 14 ) // update the next 3 rows + + jmp ( .SDONE ) + + // Reduce the 6x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 7, 4, 5, 6 ) // scale the first 3 rows by alpha + + mov ( var ( rs_c ), rdi ) // rdi = rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rdi = rs_c * sizeof ( float ) + C_STOR_BZ ( rdi, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + ZMM_REDUCE_4 ( 21, 24, 27, 30, 11 ) // xmm11 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), sum ( zmm30 )] + ZMM_REDUCE_4 ( 22, 25, 28, 31, 14 ) // xmm14 = [sum ( zmm22 ), sum ( zmm25 ), sum ( zmm28 ), sum ( zmm31 )] + + ALPHA_SCALE ( 7, 8, 11, 14 ) // scale the next 3 rows by alpha + + C_STOR_BZ_CONT ( rdi, 8, 11, 14 ) // store the next 3 rows + + label ( .SDONE ) + + // Advance to the previous 4-column block of B/C ( R-to-L within block ). + lea ( mem ( , r9, 4 ), rsi ) // rsi = 4 * cs_b_bytes + sub ( rsi, rdx ) // rdx -= 4 * cs_b + sub ( imm ( 4 ), r15 ) + cmp ( var ( j_step_start ), r15 ) + jge ( .SLOOP3X4J ) // iterate within current 16-col step + + // Finished all 4 J-iterations for this row-block; step A and C down by 6 rows. + mov ( var ( rs_c ), rdi ) // load rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rs_c *= sizeof ( float ) => rs_c *= 4 + lea ( mem ( r12, rdi, 2 ), r12 ) + lea ( mem ( r12, rdi, 4 ), r12 ) // c_ii = r12 += 6*rs_c + + lea ( mem ( r14, r8, 2 ), r14 ) + lea ( mem ( r14, r8, 4 ), r14 ) // a_ii = r14 += 6*rs_a + + dec ( r11 ) + jne ( .SLOOP3X4I ) // iterate again if ii != 0. + + // Advance j_step_start by 16 and loop if more columns remain. + mov ( var ( j_step_start ), r15 ) + sub ( imm ( 16 ), r15 ) + mov ( r15, var ( j_step_start ) ) + test ( r15, r15 ) + jns ( .SLOOP3X4J_STEP ) // loop while j_step_start >= 0 i.e. if it is not-signed + + end_asm ( + : // output operands + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_main_loop] "m" ( n_main_loop ), + [m_iter] "m" ( m_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ), + [j_step_start] "m" ( j_step_start ), + [j_step_end] "m" ( j_step_end ), + [b_step_base] "m" ( b_step_base ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) + + consider_edge_cases: + + // Control reaches here in two cases: + // 1. asm finished the full-tile rectangle and only edge work remains. + // 2. asm was skipped because m0 < 6 or n0 < 4. + // m_left dispatch: handle the bottom edge rows that remain after the asm + // kernel covers all full 6-row blocks. + // The x64 callees receive n0, but internally they only compute the 4-col + // columns [0, 4 * n_iter ). They do not consume the 1-3 column tail. + // Computes the m_left part of C. + // [ - ][ - ][ - ][ - ][ - ] + // [ - ][ - ][ - ][ - ][ - ] + // [ - ][ - ][ - ][ - ][ - ] + // [ - ][ - ][ - ][ - ][ - ] + // [ - ][ - ][ - ][ - ][ - ] + // [ - ][ - ][ - ][ - ][ - ] + // [ m_left ][ - ] + if ( m_left && n_iter ) + { + const dim_t nr_cur = n0; + const dim_t i_edge = m0 - ( dim_t )m_left; + + // Start at row 6 * m_iter, i.e. the first row not covered by asm. + float* restrict cij = c + i_edge*rs_c; + float* restrict bj = b; + float* restrict ai = a + i_edge*rs_a; + + if ( 5 == m_left ) + { + dim_t mr_cur = 5; + bli_sgemmsup_rd_zen5_asm_5x64 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 4 == m_left ) + { + const dim_t mr_cur = 4; + + bli_sgemmsup_rd_zen5_asm_4x64 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 3 == m_left ) + { + const dim_t mr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_3x64 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == m_left ) + { + const dim_t mr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_2x64 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == m_left ) + { + const dim_t mr_cur = 1; + + bli_sgemmsup_rd_zen5_asm_1x64 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + } +} + +void bli_sgemmsup_rd_zen5_asm_6x3m + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 6 rows and 3 columns of C. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 6 rows at a time; 1-5 rows fall through to m_left dispatch. + uint64_t m_iter = m0 / 6; + uint64_t m_left = m0 % 6; + + uint64_t rs_a = rs_a0; + + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // The asm microkernel only handles full 6x3 tiles. + // If m0 < 6, skip straight to the C dispatch below. + if ( m_iter == 0 ) goto consider_edge_cases; + + // Main 6x3 microkernel over the full 6x3 rectangle: + // Handles all rows in multiples of 6. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ] + // [ asm ][ asm ][ asm ] + // [ asm ][ asm ][ asm ] + // [ asm ][ asm ][ asm ] + // [ asm ][ asm ][ asm ] + // [ asm ][ asm ][ asm ] + // [ m_left ] + // ------------------------------------------------------------------------- + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + mov ( imm ( 7 ), esi ) + kmovw ( esi, K ( 2 ) ) + + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm29 ) // xmm29 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- beta + + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + mov ( var ( m_iter ), r11 ) // ii = m_iter; + label ( .SLOOP3X4I ) // Outer loop over 6-row blocks of C/A + + mov ( r12, rcx ) // rcx = base of the current 6x3 output tile in C + + // Prefetch the rows of C that will be written to in this loop + mov ( var ( rs_c ), rsi ) // rsi = rs_c + lea ( mem ( , rsi, 4 ), rsi ) // rsi = rs_c * sizeof ( float ) + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, rsi, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, rsi, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, rsi, 4 ) ) // C row 4 + lea ( mem ( rcx, rsi, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, rsi, 1 ) ) // C row 3 = rcx + 3 * rs_c + lea ( mem ( rcx, rsi, 4 ), rax ) // rax = rcx + 4 * rs_c + prefetchw0 ( mem ( rax, rsi, 1 ) ) // C row 5 = rcx + 5 * rs_c + + mov ( r14, rax ) // restart A at the top of this row-block + mov ( rdx, rbx ) // load b to rbx + + lea ( mem ( r10, r8, 2 ), rdi ) // rdi = 5 * rs_a ( rdi is reused by the store path, so recompute each iteration ) + + // zmm8-zmm28 accumulate a 6x3 tile. + INIT_ACCUM_3COL + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 3 columns of B and update the 6x3 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 3 columns of B and update the 6x3 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // The previous k-loop decomposition used iterations of 64, 32, and 8 elements, which is inefficient for k values below 32. + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm18 ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for 1-15 remaining k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), ZMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 17 MASK_KZ ( 1 ) ) ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 18 MASK_KZ ( 1 ) ) ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // For tails of <= 8 elements, use masked YMM operations for the + // remainder instead of looping element-by-element and issuing + // per-element FMAs, which is wasteful for this kernel. + // Perform a masked FMA operation on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), YMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 17 MASK_KZ ( 1 ) ) ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 18 MASK_KZ ( 1 ) ) ) + VFMA6 ( 18, 14, 15, 16, 26, 27, 28 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm30 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 6x3 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 6x3 tile. + // ZMM_REDUCE_3 folds 3 such accumulators across k and packs the 3 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..2. + // C_STOR_MASKED then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), *] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), *] + ZMM_REDUCE_3 ( 10, 13, 16, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), *] + + ALPHA_SCALE ( 29, 4, 5, 6 ) // scale the first 3 rows by alpha + + mov ( var ( rs_c ), rdi ) // rdi = rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rdi = rs_c * sizeof ( float ) + C_STOR_MASKED ( rdi, 30, 4, 5, 6 ) // update the first 3 rows of the 6x3 tile + + ZMM_REDUCE_3 ( 20, 23, 26, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), *] + ZMM_REDUCE_3 ( 21, 24, 27, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), *] + ZMM_REDUCE_3 ( 22, 25, 28, 19 ) // xmm19 = [sum ( zmm22 ), sum ( zmm25 ), sum ( zmm28 ), *] + + ALPHA_SCALE ( 29, 17, 18, 19 ) // scale the next 3 rows by alpha + + C_STOR_MASKED_CONT ( rdi, 30, 17, 18, 19 ) // update the next 3 rows + + jmp ( .SDONE ) + + // Reduce the 6x3 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_3 ( 8, 11, 14, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), *] + ZMM_REDUCE_3 ( 9, 12, 15, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), *] + ZMM_REDUCE_3 ( 10, 13, 16, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), *] + + ALPHA_SCALE ( 29, 4, 5, 6 ) // scale the first 3 rows by alpha + + mov ( var ( rs_c ), rdi ) // rdi = rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rdi = rs_c * sizeof ( float ) + C_STOR_BZ_MASKED ( rdi, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_3 ( 20, 23, 26, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), *] + ZMM_REDUCE_3 ( 21, 24, 27, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), *] + ZMM_REDUCE_3 ( 22, 25, 28, 19 ) // xmm19 = [sum ( zmm22 ), sum ( zmm25 ), sum ( zmm28 ), *] + + ALPHA_SCALE ( 29, 17, 18, 19 ) // scale the next 3 rows by alpha + + C_STOR_BZ_MASKED_CONT ( rdi, 17, 18, 19 ) // store the next 3 rows + + label ( .SDONE ) + + mov ( var ( rs_c ), rdi ) // load rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rs_c *= sizeof ( float ) => rs_c *= 4 + lea ( mem ( r12, rdi, 2 ), r12 ) + lea ( mem ( r12, rdi, 4 ), r12 ) // c_ii = r12 += 6*rs_c + + lea ( mem ( r14, r8, 2 ), r14 ) + lea ( mem ( r14, r8, 4 ), r14 ) // a_ii = r14 += 6*rs_a + + dec ( r11 ) + jne ( .SLOOP3X4I ) // iterate again if ii != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [m_iter] "m" ( m_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1", "k2" + ) + + consider_edge_cases: + + // Control reaches here in two cases: + // 1. asm finished the full 6x3 tile work and only bottom-edge rows remain. + // 2. asm was skipped because m0 < 6. + // m_left dispatch: handle the bottom edge rows that remain after the asm + // kernel covers all full 6-row blocks. + // Computes the m_left part of C. + // [ - ][ - ][ - ] + // [ - ][ - ][ - ] + // [ - ][ - ][ - ] + // [ - ][ - ][ - ] + // [ - ][ - ][ - ] + // [ - ][ - ][ - ] + // [ m_left area ] + if ( m_left ) + { + const dim_t nr_cur = n0; + const dim_t i_edge = m0 - ( dim_t )m_left; + + // Start at row 6 * m_iter, i.e. the first row not covered by asm. + float* restrict cij = c + i_edge*rs_c; + float* restrict bj = b; + float* restrict ai = a + i_edge*rs_a; + + if ( 5 == m_left ) + { + dim_t mr_cur = 5; + bli_sgemmsup_rd_zen5_asm_5x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 4 == m_left ) + { + const dim_t mr_cur = 4; + + bli_sgemmsup_rd_zen5_asm_4x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 3 == m_left ) + { + const dim_t mr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_3x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == m_left ) + { + const dim_t mr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_2x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == m_left ) + { + const dim_t mr_cur = 1; + + bli_sgemmsup_rd_zen5_asm_1x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + } +} + +void bli_sgemmsup_rd_zen5_asm_6x2m + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 6 rows and 2 columns of C. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 6 rows at a time; 1-5 rows fall through to m_left dispatch. + uint64_t m_iter = m0 / 6; + uint64_t m_left = m0 % 6; + + uint64_t rs_a = rs_a0; + + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + + float *abuf = a; + float *bbuf = b; + float *cbuf = c; + + // The asm microkernel only handles full 6x2 tiles. + // If m0 < 6, skip straight to the C dispatch below. + if ( m_iter == 0 ) goto consider_edge_cases; + + // Main 6x2 microkernel over the full 6x2 rectangle: + // Handles all rows in multiples of 6. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ] + // [ asm ][ asm ] + // [ asm ][ asm ] + // [ asm ][ asm ] + // [ asm ][ asm ] + // [ asm ][ asm ] + // [ m_left ] + // ------------------------------------------------------------------------- + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), r14 ) // load base address of a + mov ( var ( bbuf ), rdx ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm14 ) // xmm14 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm15 ) // xmm15 <- beta + + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + mov ( var ( m_iter ), r11 ) // ii = m_iter; + label ( .SLOOP3X4I ) // Outer loop over 6-row blocks of C/A + + mov ( r12, rcx ) // rcx = base of the current 6x2 output tile in C + + // Prefetch the rows of C that will be written to in this loop + mov ( var ( rs_c ), rsi ) // rsi = rs_c + lea ( mem ( , rsi, 4 ), rsi ) // rsi = rs_c * sizeof ( float ) + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, rsi, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, rsi, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, rsi, 4 ) ) // C row 4 + lea ( mem ( rcx, rsi, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, rsi, 1 ) ) // C row 3 = rcx + 3 * rs_c + lea ( mem ( rcx, rsi, 4 ), rax ) // rax = rcx + 4 * rs_c + prefetchw0 ( mem ( rax, rsi, 1 ) ) // C row 5 = rcx + 5 * rs_c + + mov ( r14, rax ) // restart A at the top of this row-block + mov ( rdx, rbx ) // load b to rbx + + lea ( mem ( r10, r8, 2 ), rdi ) // rdi = 5 * rs_a ( rdi is reused by the store path, so recompute each iteration ) + + // zmm8-zmm25 accumulate a 6x2 tile. + INIT_ACCUM_2COL + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 2 columns of B and update the 6x2 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 2 columns of B and update the 6x2 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // The previous k-loop decomposition used iterations of 64, 32, and 8 elements, which is inefficient for k values below 32. + // One full 16-float step remains before the masked k tail. + // ITER 0 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm17 ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), ZMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 17 MASK_KZ ( 1 ) ) ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // For tails of <= 8 elements, use masked YMM operations for the + // remainder instead of looping element-by-element and issuing + // per-element FMAs, which is wasteful for this kernel. + // Perform a masked FMA operation on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), YMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 17 MASK_KZ ( 1 ) ) ) + VFMA6 ( 17, 11, 12, 13, 23, 24, 25 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm15 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 6x2 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 6x2 tile. + // ZMM_REDUCE_2 folds 2 such accumulators across k and packs the 2 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..1. + // C_STOR_2_FLOATS then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), *, *] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), *, *] + ZMM_REDUCE_2 ( 10, 13, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), *, *] + + ALPHA_SCALE ( 14, 4, 5, 6 ) // scale the first 3 rows by alpha + + mov ( var ( rs_c ), rdi ) // rdi = rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rdi = rs_c * sizeof ( float ) + C_STOR_2_FLOATS ( rdi, 15, 4, 5, 6 ) // update the first 3 rows of the 6x2 tile + + ZMM_REDUCE_2 ( 20, 23, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 ), *, *] + ZMM_REDUCE_2 ( 21, 24, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 ), *, *] + ZMM_REDUCE_2 ( 22, 25, 19 ) // xmm19 = [sum ( zmm22 ), sum ( zmm25 ), *, *] + + ALPHA_SCALE ( 14, 17, 18, 19 ) // scale the next 3 rows by alpha + + C_STOR_2_FLOATS_CONT ( rdi, 15, 17, 18, 19 ) // update the next 3 rows + + jmp ( .SDONE ) + + // Reduce the 6x2 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_2 ( 8, 11, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), *, *] + ZMM_REDUCE_2 ( 9, 12, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), *, *] + ZMM_REDUCE_2 ( 10, 13, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), *, *] + + ALPHA_SCALE ( 14, 4, 5, 6 ) // scale the first 3 rows by alpha + + mov ( var ( rs_c ), rdi ) // rdi = rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rdi = rs_c * sizeof ( float ) + C_STOR_BZ_2_FLOATS ( rdi, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_2 ( 20, 23, 17 ) // xmm17 = [sum ( zmm20 ), sum ( zmm23 ), *, *] + ZMM_REDUCE_2 ( 21, 24, 18 ) // xmm18 = [sum ( zmm21 ), sum ( zmm24 ), *, *] + ZMM_REDUCE_2 ( 22, 25, 19 ) // xmm19 = [sum ( zmm22 ), sum ( zmm25 ), *, *] + + ALPHA_SCALE ( 14, 17, 18, 19 ) // scale the next 3 rows by alpha + + C_STOR_BZ_2_FLOATS_CONT ( rdi, 17, 18, 19 ) // store the next 3 rows + + label ( .SDONE ) + + mov ( var ( rs_c ), rdi ) // load rs_c + lea ( mem ( , rdi, 4 ), rdi ) // rs_c *= sizeof ( float ) => rs_c *= 4 + lea ( mem ( r12, rdi, 2 ), r12 ) + lea ( mem ( r12, rdi, 4 ), r12 ) // c_ii = r12 += 6*rs_c + + lea ( mem ( r14, r8, 2 ), r14 ) + lea ( mem ( r14, r8, 4 ), r14 ) // a_ii = r14 += 6*rs_a + + dec ( r11 ) + jne ( .SLOOP3X4I ) // iterate again if ii != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [m_iter] "m" ( m_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) + + consider_edge_cases: + + // Control reaches here in two cases: + // 1. asm finished the full 6x2 tile work and only bottom-edge rows remain. + // 2. asm was skipped because m0 < 6. + // m_left dispatch: handle the bottom edge rows that remain after the asm + // kernel covers all full 6-row blocks. + // Computes the m_left part of C. + // [ - ][ - ] + // [ - ][ - ] + // [ - ][ - ] + // [ - ][ - ] + // [ - ][ - ] + // [ - ][ - ] + // [ m_left ] + if ( m_left ) + { + const dim_t nr_cur = n0; + const dim_t i_edge = m0 - ( dim_t )m_left; + + // Start at row 6 * m_iter, i.e. the first row not covered by asm. + float* restrict cij = c + i_edge*rs_c; + float* restrict bj = b; + float* restrict ai = a + i_edge*rs_a; + + if ( 5 == m_left ) + { + dim_t mr_cur = 5; + bli_sgemmsup_rd_zen5_asm_5x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 4 == m_left ) + { + const dim_t mr_cur = 4; + + bli_sgemmsup_rd_zen5_asm_4x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 3 == m_left ) + { + const dim_t mr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_3x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == m_left ) + { + const dim_t mr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_2x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == m_left ) + { + const dim_t mr_cur = 1; + + bli_sgemmsup_rd_zen5_asm_1x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + } +} diff --git a/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64n.c b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64n.c new file mode 100644 index 000000000..155795ec0 --- /dev/null +++ b/kernels/zen5/3/sup/bli_gemmsup_rd_zen5_asm_s6x64n.c @@ -0,0 +1,2456 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +#define BLIS_ASM_SYNTAX_ATT +#include "bli_x86_asm_macros.h" + +#include "bli_gemmsup_rd_zen5_asm_s6x64.h" + +void bli_sgemmsup_rd_zen5_asm_6x64n + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This function handles m0 <= 6 rows of C. + // The asm path below is used only when m0 == 6. + uint64_t m_left = m0 % 6; + + // For m0 < 6, decompose the row edge into 3x64n, 2x64n, and finally gemv + // for the last row. + if ( m_left ) + { + float* restrict cij = c; + float* restrict bj = b; + float* restrict ai = a; + + if ( 5 <= m_left ) + { + const dim_t mr_cur = 5; + + bli_sgemmsup_rd_zen5_asm_5x64n + ( + conja, conjb, mr_cur, n0, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 4 <= m_left ) + { + const dim_t mr_cur = 4; + + bli_sgemmsup_rd_zen5_asm_4x64n + ( + conja, conjb, mr_cur, n0, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 3 <= m_left ) + { + const dim_t mr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_3x64n + ( + conja, conjb, mr_cur, n0, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 <= m_left ) + { + const dim_t mr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_2x64n + ( + conja, conjb, mr_cur, n0, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == m_left ) + { + bli_sgemv_ex + ( + BLIS_TRANSPOSE, conja, k0, n0, + alpha, bj, rs_b0, cs_b0, ai, cs_a0, + beta, cij, cs_c0, cntx, NULL + ); + } + return; + } + + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns at a time; 1-3 columns fall through to n_left dispatch. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + uint64_t cs_c = cs_c0; + + // Control reaches here when the main 4-column asm loop is skipped or when + // only the right-edge columns remain. + // n_left dispatch: handle the right edge columns that remain after the asm + // kernel covers all full 4-column blocks. + // Computes the n_left part of C. + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + if ( n_left ) + { + const dim_t mr_cur = 6; + const dim_t j_edge = n0 - ( dim_t )n_left; + + // Start at column 4 * n_iter, i.e. the first column not covered by asm. + float* restrict cij = c + j_edge*cs_c; + float* restrict ai = a; + float* restrict bj = b + j_edge*cs_b; + + if ( 3 == n_left ) + { + const dim_t nr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_6x3m + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == n_left ) + { + const dim_t nr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_6x2m + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == n_left ) + { + bli_sgemv_ex + ( + BLIS_NO_TRANSPOSE, conjb, mr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, + beta, cij, rs_c0, cntx, NULL + ); + } + } + + // The asm microkernel only handles a single full 6x4 tile in the m dimension. + if ( n_iter == 0 ) return; + + float *abuf = a; + float *bbuf = b + ( n_iter - 1 ) * 4 * cs_b; + float *cbuf = c + ( n_iter - 1 ) * 4 * cs_c; + + // Main 6x4 microkernel over the full top-left rectangle: + // Handles exactly 6 rows and columns in multiples of 4. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // ------------------------------------------------------------------------- + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + label ( .SLOOP3X4I ) // Single 6-row block + + mov ( var ( abuf ), rdx ) // load base address of a + mov ( var ( bbuf ), r14 ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + mov ( var ( n_iter ), r15 ) // jj = n_iter; + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + lea ( mem ( r10, r8, 2 ), rdi ) // rdi = 5 * rs_a + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( r12, rcx ) // rcx = base of the current 6x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, r11, 4 ) ) // C row 4 + lea ( mem ( rcx, r11, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, r11, 1 ) ) // C row 3 + lea ( mem ( rcx, r11, 4 ), rax ) // rax = rcx + 4 * rs_c + prefetchw0 ( mem ( rax, r11, 1 ) ) // C row 5 + mov ( rdx, rax ) // restart A at the top of this 4-col block + mov ( r14, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm31 accumulate a 6x4 tile. + INIT_ACCUM_4COL + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 6x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 6x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 6 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + vmovups ( ( rax, rdi, 1 ), zmm5 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 6x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), ZMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, rdi, 1 ), YMM ( 5 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 8, 9, 10, 20, 21, 22 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 11, 12, 13, 23, 24, 25 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA6 ( 6, 14, 15, 16, 26, 27, 28 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA6 ( 7, 17, 18, 19, 29, 30, 31 ) + + label ( .POST_ACCUM ) + + // Preload alpha into xmm7 to avoid redundant memory reloads. + mov ( var ( alpha ), rax ) // load address of alpha + vbroadcastss ( ( rax ), xmm7 ) // xmm7 = alpha + mov ( var ( beta ), rax ) // load address of beta + vbroadcastss ( ( rax ), xmm0 ) + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm0 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 6x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 6x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + // C_STOR then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 7, 4, 5, 6 ) // scale the first 3 rows by alpha + + // Preload beta into xmm9 now that zmm9 is free ( consumed by ZMM_REDUCE_4 ). + mov ( var ( beta ), rax ) + vbroadcastss ( ( rax ), xmm9 ) // xmm9 = beta + + C_STOR ( r11, 9, 4, 5, 6 ) // update the first 3 rows of the 6x4 tile + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + ZMM_REDUCE_4 ( 21, 24, 27, 30, 11 ) // xmm11 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), sum ( zmm30 )] + ZMM_REDUCE_4 ( 22, 25, 28, 31, 14 ) // xmm14 = [sum ( zmm22 ), sum ( zmm25 ), sum ( zmm28 ), sum ( zmm31 )] + + ALPHA_SCALE ( 7, 8, 11, 14 ) // scale the next 3 rows by alpha + + C_STOR_CONT ( r11, 9, 8, 11, 14 ) // update the next 3 rows + + jmp ( .SDONE ) + + // Reduce the 6x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 7, 4, 5, 6 ) // scale the first 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the first 3 rows without reading C + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + ZMM_REDUCE_4 ( 21, 24, 27, 30, 11 ) // xmm11 = [sum ( zmm21 ), sum ( zmm24 ), sum ( zmm27 ), sum ( zmm30 )] + ZMM_REDUCE_4 ( 22, 25, 28, 31, 14 ) // xmm14 = [sum ( zmm22 ), sum ( zmm25 ), sum ( zmm28 ), sum ( zmm31 )] + + ALPHA_SCALE ( 7, 8, 11, 14 ) // scale the next 3 rows by alpha + + C_STOR_BZ_CONT ( r11, 8, 11, 14 ) // store the next 3 rows + + label ( .SDONE ) + + sub ( imm ( 4*4 ), r12 ) // r12 -= 4 columns + lea ( mem ( , r9, 4 ), rsi ) // rsi = 4 * cs_b_bytes + sub ( rsi, r14 ) // r14 -= 4 * cs_b_bytes + + dec ( r15 ) + jne ( .SLOOP3X4J ) // iterate again if ii != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_iter] "m" ( n_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "ymm20", "ymm21", "ymm22", "ymm23", "ymm24", "ymm25", + "ymm26", "ymm27", "ymm28", "ymm29", "ymm30", "ymm31", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_5x64n + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 5 rows of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns at a time; 1-3 columns fall through to n_left dispatch. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + + uint64_t rs_a = rs_a0; + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + uint64_t cs_c = cs_c0; + + // Control reaches here when the main 4-column asm loop is skipped or when + // only the right-edge columns remain. + // n_left dispatch: handle the right edge columns that remain after the asm + // kernel covers all full 4-column blocks. + // Computes the n_left part of C. + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + if ( n_left ) + { + const dim_t mr_cur = 5; + const dim_t j_edge = n0 - ( dim_t )n_left; + + // Start at column 4 * n_iter, i.e. the first column not covered by asm. + float* restrict cij = c + j_edge*cs_c; + float* restrict ai = a; + float* restrict bj = b + j_edge*cs_b; + + if ( 3 == n_left ) + { + const dim_t nr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_5x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == n_left ) + { + const dim_t nr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_5x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == n_left ) + { + bli_sgemv_ex + ( + BLIS_NO_TRANSPOSE, conjb, mr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, + beta, cij, rs_c0, cntx, NULL + ); + } + } + + // The asm microkernel only handles full 5x4 tiles. + if ( n_iter == 0 ) return; + + float *abuf = a; + float *bbuf = b + ( n_iter - 1 ) * 4 * cs_b; + float *cbuf = c + ( n_iter - 1 ) * 4 * cs_c; + + // Main 5x4 microkernel over the full top-left rectangle: + // Handles 5 rows and columns in multiples of 4. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rdx ) // load base address of a + mov ( var ( bbuf ), r14 ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + mov ( var ( n_iter ), r15 ) // jj = n_iter; + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( r12, rcx ) // rcx = base of the current 5x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + prefetchw0 ( mem ( rcx, r11, 4 ) ) // C row 4 + lea ( mem ( rcx, r11, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, r11, 1 ) ) // C row 3 + mov ( rdx, rax ) // restart A at the top of this 4-col block + mov ( r14, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm19 accumulate a 5x4 tile. + INIT_ACCUM_5x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + vmovups ( ( rax, r8, 4 ), zmm4 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( ( rbx, r9, 2 ), zmm6 ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( ( rbx, r13, 1 ), zmm7 ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), ZMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 4 ), YMM ( 4 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 8, 9, 10, 20, 26 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 11, 12, 13, 21, 27 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA5 ( 6, 14, 15, 16, 23, 29 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA5 ( 7, 17, 18, 19, 24, 30 ) + + label ( .POST_ACCUM ) + + // Preload alpha into xmm7 to avoid redundant memory reloads. + mov ( var ( alpha ), rax ) // load address of alpha + vbroadcastss ( ( rax ), xmm7 ) // xmm7 = alpha + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 3x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 3x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + // C_STOR then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 7, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR ( r11, 31, 4, 5, 6 ) // update the 3x4 tile in C + + ZMM_REDUCE_4 ( 20, 21, 23, 24, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm21 ), sum ( zmm23 ), sum ( zmm24 )] + ZMM_REDUCE_4 ( 26, 27, 29, 30, 11 ) // xmm11 = [sum ( zmm26 ), sum ( zmm27 ), sum ( zmm29 ), sum ( zmm30 )] + + ALPHA_SCALE2 ( 7, 8, 11 ) // scale the next 2 rows by alpha + + C_STOR2_CONT ( r11, 31, 8, 11 ) // update the next 2 rows + + jmp ( .SDONE ) + + // Reduce the 3x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 7, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the 3x4 tile without reading C + + ZMM_REDUCE_4 ( 20, 21, 23, 24, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm21 ), sum ( zmm23 ), sum ( zmm24 )] + ZMM_REDUCE_4 ( 26, 27, 29, 30, 11 ) // xmm11 = [sum ( zmm26 ), sum ( zmm27 ), sum ( zmm29 ), sum ( zmm30 )] + + ALPHA_SCALE2 ( 7, 8, 11 ) // scale the next 2 rows by alpha + + C_STOR_BZ2_CONT ( r11, 8, 11 ) // store the next 2 rows + + label ( .SDONE ) + + sub ( imm ( 4*4 ), r12 ) // r12 -= 4 columns + + lea ( mem ( , r9, 4 ), rsi ) // rsi = 4 * cs_b_bytes + sub ( rsi, r14 ) // r14 -= 4 * cs_b_bytes + + dec ( r15 ) + jne ( .SLOOP3X4J ) // iterate again if jj != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_iter] "m" ( n_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_4x64n + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 4 rows of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns at a time; 1-3 columns fall through to n_left dispatch. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + + uint64_t rs_a = rs_a0; + + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + uint64_t cs_c = cs_c0; + + // Control reaches here when the main 4-column asm loop is skipped or when + // only the right-edge columns remain. + // n_left dispatch: handle the right edge columns that remain after the asm + // kernel covers all full 4-column blocks. + // Computes the n_left part of C. + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + if ( n_left ) + { + const dim_t mr_cur = 4; + const dim_t j_edge = n0 - ( dim_t )n_left; + + // Start at column 4 * n_iter, i.e. the first column not covered by asm. + float* restrict cij = c + j_edge*cs_c; + float* restrict ai = a; + float* restrict bj = b + j_edge*cs_b; + + if ( 3 == n_left ) + { + const dim_t nr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_4x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == n_left ) + { + const dim_t nr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_4x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == n_left ) + { + bli_sgemv_ex + ( + BLIS_NO_TRANSPOSE, conjb, mr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, + beta, cij, rs_c0, cntx, NULL + ); + } + } + + // The asm microkernel only handles full 4x4 tiles. + if ( n_iter == 0 ) return; + + float *abuf = a; + float *bbuf = b + ( n_iter - 1 ) * 4 * cs_b; + float *cbuf = c + ( n_iter - 1 ) * 4 * cs_c; + + // Main 4x4 microkernel over the full top-left rectangle: + // Handles 4 rows and columns in multiples of 4. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + lea ( mem ( r8, r8, 2 ), r10 ) // r10 = 3 * rs_a + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rdx ) // load base address of a + mov ( var ( bbuf ), r14 ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + mov ( var ( n_iter ), r15 ) // jj = n_iter; + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( r12, rcx ) // rcx = base of the current 4x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + lea ( mem ( rcx, r11, 2 ), rax ) // rax = rcx + 2 * rs_c + prefetchw0 ( mem ( rax, r11, 1 ) ) // C row 3 + mov ( rdx, rax ) // restart A at the top of this 4-col block + mov ( r14, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm20 accumulate a 4x4 tile. + INIT_ACCUM_4x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 4 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 4x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + vmovups ( ( rax, r10, 1 ), zmm3 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( ( rbx, r9, 2 ), zmm21 ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( ( rbx, r13, 1 ), zmm6 ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), ZMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 21 MASK_KZ ( 1 ) ) ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + // Perform a masked FMA operation on the remaining elements + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r10, 1 ), YMM ( 3 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 8, 9, 10, 20 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA4 ( 7, 11, 12, 13, 23 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 21 MASK_KZ ( 1 ) ) ) + VFMA4 ( 21, 14, 15, 16, 26 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA4 ( 6, 17, 18, 19, 29 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 3x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 3x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + // C_STOR then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR ( r11, 31, 4, 5, 6 ) // update the 3x4 tile in C + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + + ALPHA_SCALE1 ( 30, 8 ) // scale the 4 elements by alpha + + C_STOR1_CONT ( r11, 31, 8 ) // update the next row + + jmp ( .SDONE ) + + // Reduce the 3x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the 3x4 tile without reading C + + ZMM_REDUCE_4 ( 20, 23, 26, 29, 8 ) // xmm8 = [sum ( zmm20 ), sum ( zmm23 ), sum ( zmm26 ), sum ( zmm29 )] + + ALPHA_SCALE1 ( 30, 8 ) // scale the 4 elements by alpha + + C_STOR_BZ1_CONT ( r11, 8 ) // store the next row + + label ( .SDONE ) + + sub ( imm ( 4*4 ), r12 ) // r12 -= 4 columns + + lea ( mem ( , r9, 4 ), rsi ) // rsi = 4 * cs_b_bytes + sub ( rsi, r14 ) // r14 -= 4 * cs_b_bytes + + dec ( r15 ) + jne ( .SLOOP3X4J ) // iterate again if jj != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_iter] "m" ( n_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_3x64n + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 3 rows of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns at a time; 1-3 columns fall through to n_left dispatch. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + + uint64_t rs_a = rs_a0; + + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + uint64_t cs_c = cs_c0; + + // Control reaches here when the main 4-column asm loop is skipped or when + // only the right-edge columns remain. + // n_left dispatch: handle the right edge columns that remain after the asm + // kernel covers all full 4-column blocks. + // Computes the n_left part of C. + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + if ( n_left ) + { + const dim_t mr_cur = 3; + const dim_t j_edge = n0 - ( dim_t )n_left; + + // Start at column 4 * n_iter, i.e. the first column not covered by asm. + float* restrict cij = c + j_edge*cs_c; + float* restrict ai = a; + float* restrict bj = b + j_edge*cs_b; + + if ( 3 == n_left ) + { + const dim_t nr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_3x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == n_left ) + { + const dim_t nr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_3x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == n_left ) + { + bli_sgemv_ex + ( + BLIS_NO_TRANSPOSE, conjb, mr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, + beta, cij, rs_c0, cntx, NULL + ); + } + } + + // The asm microkernel only handles full 3x4 tiles. + if ( n_iter == 0 ) return; + + float *abuf = a; + float *bbuf = b + ( n_iter - 1 ) * 4 * cs_b; + float *cbuf = c + ( n_iter - 1 ) * 4 * cs_c; + + // Main 3x4 microkernel over the full top-left rectangle: + // Handles 3 rows and columns in multiples of 4. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rdx ) // load base address of a + mov ( var ( bbuf ), r14 ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + mov ( var ( n_iter ), r15 ) // jj = n_iter; + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( r12, rcx ) // rcx = base of the current 3x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + prefetchw0 ( mem ( rcx, r11, 2 ) ) // C row 2 + mov ( rdx, rax ) // restart A at the top of this 4-col block + mov ( r14, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm19 accumulate a 3x4 tile. + INIT_ACCUM_3x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 3 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + vmovups ( ( rax, r8, 2 ), zmm2 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 3x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA3 ( 21, 17, 18, 19 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), ZMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 20 MASK_KZ ( 1 ) ) ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 21 MASK_KZ ( 1 ) ) ) + VFMA3 ( 21, 17, 18, 19 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 2 ), YMM ( 2 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA3 ( 6, 8, 9, 10 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA3 ( 7, 11, 12, 13 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 20 MASK_KZ ( 1 ) ) ) + VFMA3 ( 20, 14, 15, 16 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 21 MASK_KZ ( 1 ) ) ) + VFMA3 ( 21, 17, 18, 19 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 3x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 3x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5/xmm6 <- rows 0..2, cols 0..3. + // C_STOR then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR ( r11, 31, 4, 5, 6 ) // update the 3x4 tile in C + + jmp ( .SDONE ) + + // Reduce the 3x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + ZMM_REDUCE_4 ( 10, 13, 16, 19, 6 ) // xmm6 = [sum ( zmm10 ), sum ( zmm13 ), sum ( zmm16 ), sum ( zmm19 )] + + ALPHA_SCALE ( 30, 4, 5, 6 ) // scale the 3 rows by alpha + + C_STOR_BZ ( r11, 4, 5, 6 ) // store the 3x4 tile without reading C + + label ( .SDONE ) + + sub ( imm ( 4*4 ), r12 ) // r12 -= 4 columns + + lea ( mem ( , r9, 4 ), rsi ) // rsi = 4 * cs_b_bytes + sub ( rsi, r14 ) // r14 -= 4 * cs_b_bytes + + dec ( r15 ) + jne ( .SLOOP3X4J ) // iterate again if jj != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_iter] "m" ( n_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm16", "ymm17", "ymm18", "ymm19", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} + +void bli_sgemmsup_rd_zen5_asm_2x64n + ( + conj_t conja, + conj_t conjb, + dim_t m0, + dim_t n0, + dim_t k0, + float* restrict alpha, + float* restrict a, inc_t rs_a0, inc_t cs_a0, + float* restrict b, inc_t rs_b0, inc_t cs_b0, + float* restrict beta, + float* restrict c, inc_t rs_c0, inc_t cs_c0, + auxinfo_t* restrict data, + cntx_t* restrict cntx + ) +{ + // This kernel handles 2 rows of C. + // Decompose k into 64-float, 32-float, 16-float, and masked tail work. + uint64_t k_iter64 = k0 / 64; + uint64_t k_left64 = k0 % 64; + uint64_t k_iter32 = k_left64 / 32; + uint64_t k_left32 = k_left64 % 32; + uint64_t k_iter16 = k_left32 / 16; + uint64_t k_left1 = k_left32 % 16; + int32_t iter_1_mask = ( 1 << k_left1 ) - 1; + + // Main loop handles 4 columns at a time; 1-3 columns fall through to n_left dispatch. + uint64_t n_iter = n0 / 4; + uint64_t n_left = n0 % 4; + + uint64_t rs_a = rs_a0; + + uint64_t cs_b = cs_b0; + uint64_t rs_c = rs_c0; + uint64_t cs_c = cs_c0; + + // Control reaches here when the main 4-column asm loop is skipped or when + // only the right-edge columns remain. + // n_left dispatch: handle the right edge columns that remain after the asm + // kernel covers all full 4-column blocks. + // Computes the n_left part of C. + // [ - ][ - ][ - ][ - ][ n_left ] + // [ - ][ - ][ - ][ - ][ n_left ] + if ( n_left ) + { + const dim_t mr_cur = 2; + const dim_t j_edge = n0 - ( dim_t )n_left; + + // Start at column 4 * n_iter, i.e. the first column not covered by asm. + float* restrict cij = c + j_edge*cs_c; + float* restrict ai = a; + float* restrict bj = b + j_edge*cs_b; + + if ( 3 == n_left ) + { + const dim_t nr_cur = 3; + + bli_sgemmsup_rd_zen5_asm_2x3 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 2 == n_left ) + { + const dim_t nr_cur = 2; + + bli_sgemmsup_rd_zen5_asm_2x2 + ( + conja, conjb, mr_cur, nr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, cs_b0, + beta, cij, rs_c0, cs_c0, data, cntx + ); + } + else if ( 1 == n_left ) + { + bli_sgemv_ex + ( + BLIS_NO_TRANSPOSE, conjb, mr_cur, k0, + alpha, ai, rs_a0, cs_a0, bj, rs_b0, + beta, cij, rs_c0, cntx, NULL + ); + } + } + + // The asm microkernel only handles full 2x4 tiles. + if ( n_iter == 0 ) return; + + float *abuf = a; + float *bbuf = b + ( n_iter - 1 ) * 4 * cs_b; + float *cbuf = c + ( n_iter - 1 ) * 4 * cs_c; + + // Main 2x4 microkernel over the full top-left rectangle: + // Handles 2 rows and columns in multiples of 4. + // ------------------------------------------------------------------------- + // Computes the asm part of C. + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + // [ asm ][ asm ][ asm ][ asm ][ n_left ] + begin_asm() + + mov ( var ( rs_a ), r8 ) // load rs_a + lea ( mem ( , r8, 4 ), r8 ) // rs_a *= sizeof ( dt ) => rs_a *= 4 + mov ( var ( cs_b ), r9 ) // load cs_b + lea ( mem ( , r9, 4 ), r9 ) // cs_b *= sizeof ( dt ) => cs_b *= 4 + lea ( mem ( r9, r9, 2 ), r13 ) // r13 = 3 * cs_b in bytes + + // preload alpha and beta into vector registers that are unused in this kernel + mov ( var ( alpha ), rsi ) // load address of alpha + vbroadcastss ( ( rsi ), xmm30 ) // xmm30 <- alpha + mov ( var ( beta ), rsi ) // load address of beta + vbroadcastss ( ( rsi ), xmm31 ) // xmm31 <- beta + + mov ( var ( iter_1_mask ), esi ) // k1 = lane mask for the final k tail + kmovw ( esi, K ( 1 ) ) + + mov ( var ( abuf ), rdx ) // load base address of a + mov ( var ( bbuf ), r14 ) // load base address of b + mov ( var ( cbuf ), r12 ) // load base address of c + + mov ( var ( n_iter ), r15 ) // jj = n_iter; + mov ( var ( rs_c ), r11 ) // r11 = rs_c + lea ( mem ( , r11, 4 ), r11 ) // r11 = rs_c * sizeof ( float ) + label ( .SLOOP3X4J ) // Inner loop over 4-column output tiles + + mov ( r12, rcx ) // rcx = base of the current 2x4 output tile in C + prefetchw0 ( mem ( rcx ) ) // C row 0 + prefetchw0 ( mem ( rcx, r11, 1 ) ) // C row 1 + mov ( rdx, rax ) // restart A at the top of this 4-col block + mov ( r14, rbx ) // restart B at the top of this 4-col block + + // zmm8-zmm18 accumulate a 2x4 tile. + INIT_ACCUM_2x4 + + mov ( var ( k_iter64 ), rsi ) // number of 64-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_32 ) + + label ( .K_LOOP_ITER64 ) + + // Each unrolled iteration consumes 16 floats from each A row and 16 floats from each B column. + // Four such iterations make one 64-float k block. + // ITER 0 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 2x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 2 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 3 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + dec ( rsi ) + jne ( .K_LOOP_ITER64 ) + + label ( .CONSIDER_K_ITER_32 ) + + mov ( var ( k_iter32 ), rsi ) // number of remaining 32-float k blocks + test ( rsi, rsi ) + je ( .CONSIDER_K_ITER_16 ) + + // Two 16-float iterations cover the 32-float remainder block. + // ITER 0 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 2x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + // ITER 1 + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load the next 16-float B vectors and continue accumulating. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_ITER_16 ) + mov ( var ( k_iter16 ), rsi ) + test ( rsi, rsi ) + je ( .CONSIDER_K_LEFT_1 ) + + // One full 16-float step remains before the masked k tail. + // ITER 0 + // Load one 16-float vector from each of the 2 rows of A. + vmovups ( ( rax ), zmm0 ) + vmovups ( ( rax, r8, 1 ), zmm1 ) + add ( imm ( 16*4 ), rax ) + + // Load one 16-float vector from each of the 4 columns of B and update the 2x4 accumulators. + vmovups ( ( rbx ), zmm6 ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( ( rbx, r9, 1 ), zmm7 ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( ( rbx, r9, 2 ), zmm20 ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( ( rbx, r13, 1 ), zmm21 ) + VFMA2 ( 21, 17, 18 ) + + add ( imm ( 16*4 ), rbx ) + + label ( .CONSIDER_K_LEFT_1 ) + mov ( var ( k_left1 ), rsi ) + test ( rsi, rsi ) + je ( .POST_ACCUM ) + + // In the case where we need to only compute on floats + // which fit in the ymm register, it is better to + // operate on masked ymm registers in this case because + // in practice this is faster on zen5 + cmp ( imm ( 8 ), rsi ) + jle ( .K_FLOATS_LEFT_LE_8 ) + + label ( .K_FLOATS_LEFT_GT_8 ) + // Masked ZMM tail for the final 1-15 k values. + vmovups ( mem ( rax ), ZMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), ZMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), ZMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), ZMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( mem ( rbx, r9, 2 ), ZMM ( 20 MASK_KZ ( 1 ) ) ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( mem ( rbx, r13, 1 ), ZMM ( 21 MASK_KZ ( 1 ) ) ) + VFMA2 ( 21, 17, 18 ) + + // unconditional branch to end of the loop after + // the computation of the case processing >8 floats + jmp ( .POST_ACCUM ) + + label ( .K_FLOATS_LEFT_LE_8 ) + // When operating on <= 8 remaining elements, use masked YMM + // registers for the tail path rather than handling each element + // individually. This avoids a wasteful element-by-element loop + // and keeps the tail processing as a single masked vector FMA + // sequence on the remaining elements. + vmovups ( mem ( rax ), YMM ( 0 MASK_KZ ( 1 ) ) ) + vmovups ( mem ( rax, r8, 1 ), YMM ( 1 MASK_KZ ( 1 ) ) ) + + vmovups ( mem ( rbx ), YMM ( 6 MASK_KZ ( 1 ) ) ) + VFMA2 ( 6, 8, 9 ) + + vmovups ( mem ( rbx, r9, 1 ), YMM ( 7 MASK_KZ ( 1 ) ) ) + VFMA2 ( 7, 11, 12 ) + + vmovups ( mem ( rbx, r9, 2 ), YMM ( 20 MASK_KZ ( 1 ) ) ) + VFMA2 ( 20, 14, 15 ) + + vmovups ( mem ( rbx, r13, 1 ), YMM ( 21 MASK_KZ ( 1 ) ) ) + VFMA2 ( 21, 17, 18 ) + + label ( .POST_ACCUM ) + + vxorps ( xmm1, xmm1, xmm1 ) + vucomiss ( xmm1, xmm31 ) // branch on beta == 0 to skip reading C + je ( .POST_ACCUM_STOR_BZ ) + + // Reduce the 2x4 accumulators, scale by alpha, then update C with beta * C + accum. + label ( .POST_ACCUM_STOR ) + + // Each zmm accumulator holds 16 partial sums for one C ( i,j ) in the + // current 2x4 tile. + // ZMM_REDUCE_4 folds 4 such accumulators across k and packs the 4 final + // column results for one row into one xmm register. + // xmm4/xmm5 <- rows 0..1, cols 0..3. + // C_STOR2 then does beta * C + accum and writes those row vectors to C. + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR2 ( r11, 31, 4, 5 ) // update the 2x4 tile in C + + jmp ( .SDONE ) + + // Reduce the 2x4 accumulators, scale by alpha, then store directly when beta == 0. + label ( .POST_ACCUM_STOR_BZ ) + + ZMM_REDUCE_4 ( 8, 11, 14, 17, 4 ) // xmm4 = [sum ( zmm8 ), sum ( zmm11 ), sum ( zmm14 ), sum ( zmm17 )] + ZMM_REDUCE_4 ( 9, 12, 15, 18, 5 ) // xmm5 = [sum ( zmm9 ), sum ( zmm12 ), sum ( zmm15 ), sum ( zmm18 )] + + ALPHA_SCALE2 ( 30, 4, 5 ) // scale the 2 rows by alpha + + C_STOR_BZ2 ( r11, 4, 5 ) // store the 2x4 tile without reading C + + label ( .SDONE ) + + sub ( imm ( 4*4 ), r12 ) // r12 -= 4 columns + lea ( mem ( , r9, 4 ), rsi ) // rsi = 4 * cs_b_bytes + sub ( rsi, r14 ) // r14 -= 4 * cs_b_bytes + + dec ( r15 ) + jne ( .SLOOP3X4J ) // iterate again if jj != 0. + + end_asm ( + : // output operands ( none ) + : // input operands + [iter_1_mask] "m" ( iter_1_mask ), + [k_iter64] "m" ( k_iter64 ), + [k_iter32] "m" ( k_iter32 ), + [k_iter16] "m" ( k_iter16 ), + [k_left1] "m" ( k_left1 ), + [rs_a] "m" ( rs_a ), + [cs_b] "m" ( cs_b ), + [alpha] "m" ( alpha ), + [beta] "m" ( beta ), + [rs_c] "m" ( rs_c ), + [n_iter] "m" ( n_iter ), + [abuf] "m" ( abuf ), + [bbuf] "m" ( bbuf ), + [cbuf] "m" ( cbuf ) + : // register clobber list + "rax", "rbx", "rcx", "rdx", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", + "ymm7", "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", + "ymm14", "ymm15", "ymm17", "ymm18", + "zmm0", "zmm1", "zmm2", "zmm3", + "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9", "zmm10", + "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", + "zmm16", "zmm17", "zmm18", "zmm19", + "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", + "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", + "memory", "k1" + ) +} diff --git a/kernels/zen5/bli_kernels_zen5.h b/kernels/zen5/bli_kernels_zen5.h index bd0445014..49d1b8110 100644 --- a/kernels/zen5/bli_kernels_zen5.h +++ b/kernels/zen5/bli_kernels_zen5.h @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2024 - 2025, Advanced Micro Devices, Inc. All rights reserved. + Copyright (C) 2024 - 2026, Advanced Micro Devices, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -32,6 +32,8 @@ */ +#include "bli_gemm_tiny_zen5.h" + // DCOPYV kernel COPYV_KER_PROT( double, d, copyv_zen5_asm ) @@ -45,6 +47,32 @@ GEMMSUP_KER_PROT( double, d, gemmsup_cv_zen5_asm_24x3m) GEMMSUP_KER_PROT( double, d, gemmsup_cv_zen5_asm_24x2m) GEMMSUP_KER_PROT( double, d, gemmsup_cv_zen5_asm_24x1m) + +// sgemm sup RD kernels +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_6x64m ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_6x3m ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_6x2m ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_5x64 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_4x64 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_3x64 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_2x64 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_1x64 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_5x3 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_4x3 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_3x3 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_2x3 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_1x3 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_5x2 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_4x2 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_3x2 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_2x2 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_1x2 ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_6x64n ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_5x64n ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_4x64n ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_3x64n ) +GEMMSUP_KER_PROT( float, s, gemmsup_rd_zen5_asm_2x64n ) + // threshold functions bool bli_cntx_gemmsup_thresh_is_met_zen5 (